如何写入 json 大小限制(100KB)的文件,一旦大小超过,写入新文件?

how to write json file with size limit (100KB) and once size cross, write new file?

我有一个 blocking collection 被一些 app1.

填满了数据

我已经订阅了 blocking collection 并且需要写一个包含以下情况的文件,

  1. 开始写入文件。
  2. 如果文件大小超过 100kb,关闭第一个文件并开始一个新文件。
  3. 如果 app1 没有数据,假设 1 分钟,然后关闭文件。

目前使用以下代码,我只能将每个 blocking collection 写入每个文件,如何继续我的上述要求,请建议。

class Program
{
    private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
    private static void Producer()
    {
        int ctr = 1;
        while (true)
        {
            messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
            Thread.Sleep(1000);
            ctr++;
        }
    }

    private static void Consumer()
    {
        foreach (var message in messages.GetConsumingEnumerable())
        {
            Console.WriteLine(JsonConvert.SerializeObject(message));

            using (var streamWriter = File.CreateText(Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json")))
            {
                using (var writer = new JsonTextWriter(streamWriter))
                {
                    writer.Formatting = Formatting.Indented;
                    writer.WriteStartObject();
                    writer.WritePropertyName("Data");
                    writer.WriteStartArray();
                    writer.WriteRawValue(JsonConvert.SerializeObject(message));
                    writer.WriteEndArray();
                    writer.WriteEndObject();
                }
            }
        }
    }
    static void Main(string[] args)
    {
        var producer = Task.Factory.StartNew(() => Producer());
        var consumer = Task.Factory.StartNew(() => Consumer());
        Console.Read();
    }
}

这会将任意大小的消息分成不同的 JSON 文件。您需要使用 maxchar 指定大小。在此之前,您必须像这样检查最后一个文件的大小,如果创建新文件并分割消息,则必须传递相同的文件名和新大小。

using System;
using System.IO;
using System;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;



using System.Collections;


using Newtonsoft.Json;
using System.Runtime.Remoting.Messaging;
using System.Text;

namespace Program
{
    class Program
    {
        public static string last_path = "";
        public static readonly string BYE = "bye";
        private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
        private static void Producer()
        {
            int ctr = 1;
            while (true)
            {
                messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
                Thread.Sleep(1000);
                ctr++;
            }
        }

        private static void Consumer()
        {
            foreach (var message in messages.GetConsumingEnumerable())
            {
                Console.WriteLine(JsonConvert.SerializeObject(message));
                string json = JsonConvert.SerializeObject(message);
                int maxchar = 102400;
                if (last_path != "")
                {
                    long length = new FileInfo(last_path).Length;
                    if (length < maxchar)
                    {
                        maxchar = maxchar - unchecked((int)length);
                        dividefile(last_path, maxchar, json);
                    }

                    else

                    {
                        dividefile("", maxchar, json);

                    }
                }
                else
                {
                    dividefile("", maxchar, json);

                }

            }
        }

        public static void dividefile(string path, int maxchar, string message)
        {

            //FileStream fileStream = new FileStream(yourfile, FileMode.Open, FileAccess.Read);
            byte[] byteArray = Encoding.UTF8.GetBytes(message);

            MemoryStream stream = new MemoryStream(byteArray);

            using (StreamReader streamReader = new StreamReader(stream))
            {
                Int64 x1 = stream.Length;
                char[] fileContents = new char[maxchar];
                int charsRead = streamReader.Read(fileContents, 0, maxchar);

                // Can't do much with 0 bytes
                if (charsRead == 0)
                    throw new Exception("File is 0 bytes");

                while (charsRead > 0)
                {
                    x1 = x1 - maxchar;

                    if (x1 > 0)
                    {
                        string s = new string(fileContents);
                        if (path == "")
                        {
                            last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
                            path = "";
                        }
                        else
                        {
                            last_path = path;
                        }
                        AppendTransaction(last_path, s);
                        charsRead = streamReader.Read(fileContents, 0, maxchar);
                    }
                    else
                    {
                        int m = (int)(((x1 + maxchar) % maxchar));
                        string messagechunk = new string(fileContents, 0, m);
                        if (path == "")
                        {
                            last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
                            path = "";
                        }
                        else
                        {
                            last_path = path;
                        }

                        AppendTransaction(last_path, messagechunk);

                        charsRead = streamReader.Read(fileContents, 0, m);
                    }
                }
            }

        }

        private static void AppendTransaction(string path , string  transaction)
        {
             string filename = path;
            bool firstTransaction = !File.Exists(filename);

            JsonSerializer ser = new JsonSerializer();
            ser.Formatting = Formatting.Indented;
            ser.TypeNameHandling = TypeNameHandling.Auto;

            using (var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
            {
                Encoding enc = firstTransaction ? new UTF8Encoding(true) : new UTF8Encoding(false);

                using (var sw = new StreamWriter(fs, enc))
                using (var jtw = new JsonTextWriter(sw))
                {
                    if (firstTransaction)
                    {
                        sw.Write("[");
                        sw.Flush();
                    }

                    else
                    {
                        fs.Seek(-Encoding.UTF8.GetByteCount("]"), SeekOrigin.End);
                        sw.Write(",");
                        sw.Flush();
                    }

                    ser.Serialize(jtw, transaction);
                    sw.Write(']');
                }
            }


        }
        static void Main(string[] args)
        {
            var producer = Task.Factory.StartNew(() => Producer());
            var consumer = Task.Factory.StartNew(() => Consumer());
            Console.Read();
        }
        class Message
        {
            public int ProductorThreadID { get; set; }
            public int CustomerThreadID { get; set; }
            public string key { get; set; }
            public string content { get; set; }
            public string Name { get; internal set; }
            public int Id { get; internal set; }

            public bool endThread()
            {
                return string.Compare(key, Program.BYE) == 0;
            }

            public string ToString(bool isProductor)
            {
                return string.Format("{0} Thread ID {1} : {2}", isProductor ? "Productor" : "Customer",
                                                                isProductor ? ProductorThreadID.ToString() : CustomerThreadID.ToString(),
                                                                content);
            }
        }
    }
}