如何写入 json 大小限制(100KB)的文件,一旦大小超过,写入新文件?
how to write json file with size limit (100KB) and once size cross, write new file?
我有一个 blocking
collection 被一些 app1
.
填满了数据
我已经订阅了 blocking
collection 并且需要写一个包含以下情况的文件,
- 开始写入文件。
- 如果文件大小超过 100kb,关闭第一个文件并开始一个新文件。
- 如果
app1
没有数据,假设 1 分钟,然后关闭文件。
目前使用以下代码,我只能将每个 blocking
collection 写入每个文件,如何继续我的上述要求,请建议。
class Program
{
private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
private static void Producer()
{
int ctr = 1;
while (true)
{
messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
Thread.Sleep(1000);
ctr++;
}
}
private static void Consumer()
{
foreach (var message in messages.GetConsumingEnumerable())
{
Console.WriteLine(JsonConvert.SerializeObject(message));
using (var streamWriter = File.CreateText(Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json")))
{
using (var writer = new JsonTextWriter(streamWriter))
{
writer.Formatting = Formatting.Indented;
writer.WriteStartObject();
writer.WritePropertyName("Data");
writer.WriteStartArray();
writer.WriteRawValue(JsonConvert.SerializeObject(message));
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
}
static void Main(string[] args)
{
var producer = Task.Factory.StartNew(() => Producer());
var consumer = Task.Factory.StartNew(() => Consumer());
Console.Read();
}
}
这会将任意大小的消息分成不同的 JSON 文件。您需要使用 maxchar 指定大小。在此之前,您必须像这样检查最后一个文件的大小,如果创建新文件并分割消息,则必须传递相同的文件名和新大小。
using System;
using System.IO;
using System;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Collections;
using Newtonsoft.Json;
using System.Runtime.Remoting.Messaging;
using System.Text;
namespace Program
{
class Program
{
public static string last_path = "";
public static readonly string BYE = "bye";
private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
private static void Producer()
{
int ctr = 1;
while (true)
{
messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
Thread.Sleep(1000);
ctr++;
}
}
private static void Consumer()
{
foreach (var message in messages.GetConsumingEnumerable())
{
Console.WriteLine(JsonConvert.SerializeObject(message));
string json = JsonConvert.SerializeObject(message);
int maxchar = 102400;
if (last_path != "")
{
long length = new FileInfo(last_path).Length;
if (length < maxchar)
{
maxchar = maxchar - unchecked((int)length);
dividefile(last_path, maxchar, json);
}
else
{
dividefile("", maxchar, json);
}
}
else
{
dividefile("", maxchar, json);
}
}
}
public static void dividefile(string path, int maxchar, string message)
{
//FileStream fileStream = new FileStream(yourfile, FileMode.Open, FileAccess.Read);
byte[] byteArray = Encoding.UTF8.GetBytes(message);
MemoryStream stream = new MemoryStream(byteArray);
using (StreamReader streamReader = new StreamReader(stream))
{
Int64 x1 = stream.Length;
char[] fileContents = new char[maxchar];
int charsRead = streamReader.Read(fileContents, 0, maxchar);
// Can't do much with 0 bytes
if (charsRead == 0)
throw new Exception("File is 0 bytes");
while (charsRead > 0)
{
x1 = x1 - maxchar;
if (x1 > 0)
{
string s = new string(fileContents);
if (path == "")
{
last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
path = "";
}
else
{
last_path = path;
}
AppendTransaction(last_path, s);
charsRead = streamReader.Read(fileContents, 0, maxchar);
}
else
{
int m = (int)(((x1 + maxchar) % maxchar));
string messagechunk = new string(fileContents, 0, m);
if (path == "")
{
last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
path = "";
}
else
{
last_path = path;
}
AppendTransaction(last_path, messagechunk);
charsRead = streamReader.Read(fileContents, 0, m);
}
}
}
}
private static void AppendTransaction(string path , string transaction)
{
string filename = path;
bool firstTransaction = !File.Exists(filename);
JsonSerializer ser = new JsonSerializer();
ser.Formatting = Formatting.Indented;
ser.TypeNameHandling = TypeNameHandling.Auto;
using (var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
Encoding enc = firstTransaction ? new UTF8Encoding(true) : new UTF8Encoding(false);
using (var sw = new StreamWriter(fs, enc))
using (var jtw = new JsonTextWriter(sw))
{
if (firstTransaction)
{
sw.Write("[");
sw.Flush();
}
else
{
fs.Seek(-Encoding.UTF8.GetByteCount("]"), SeekOrigin.End);
sw.Write(",");
sw.Flush();
}
ser.Serialize(jtw, transaction);
sw.Write(']');
}
}
}
static void Main(string[] args)
{
var producer = Task.Factory.StartNew(() => Producer());
var consumer = Task.Factory.StartNew(() => Consumer());
Console.Read();
}
class Message
{
public int ProductorThreadID { get; set; }
public int CustomerThreadID { get; set; }
public string key { get; set; }
public string content { get; set; }
public string Name { get; internal set; }
public int Id { get; internal set; }
public bool endThread()
{
return string.Compare(key, Program.BYE) == 0;
}
public string ToString(bool isProductor)
{
return string.Format("{0} Thread ID {1} : {2}", isProductor ? "Productor" : "Customer",
isProductor ? ProductorThreadID.ToString() : CustomerThreadID.ToString(),
content);
}
}
}
}
我有一个 blocking
collection 被一些 app1
.
我已经订阅了 blocking
collection 并且需要写一个包含以下情况的文件,
- 开始写入文件。
- 如果文件大小超过 100kb,关闭第一个文件并开始一个新文件。
- 如果
app1
没有数据,假设 1 分钟,然后关闭文件。
目前使用以下代码,我只能将每个 blocking
collection 写入每个文件,如何继续我的上述要求,请建议。
class Program
{
private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
private static void Producer()
{
int ctr = 1;
while (true)
{
messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
Thread.Sleep(1000);
ctr++;
}
}
private static void Consumer()
{
foreach (var message in messages.GetConsumingEnumerable())
{
Console.WriteLine(JsonConvert.SerializeObject(message));
using (var streamWriter = File.CreateText(Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json")))
{
using (var writer = new JsonTextWriter(streamWriter))
{
writer.Formatting = Formatting.Indented;
writer.WriteStartObject();
writer.WritePropertyName("Data");
writer.WriteStartArray();
writer.WriteRawValue(JsonConvert.SerializeObject(message));
writer.WriteEndArray();
writer.WriteEndObject();
}
}
}
}
static void Main(string[] args)
{
var producer = Task.Factory.StartNew(() => Producer());
var consumer = Task.Factory.StartNew(() => Consumer());
Console.Read();
}
}
这会将任意大小的消息分成不同的 JSON 文件。您需要使用 maxchar 指定大小。在此之前,您必须像这样检查最后一个文件的大小,如果创建新文件并分割消息,则必须传递相同的文件名和新大小。
using System;
using System.IO;
using System;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Collections;
using Newtonsoft.Json;
using System.Runtime.Remoting.Messaging;
using System.Text;
namespace Program
{
class Program
{
public static string last_path = "";
public static readonly string BYE = "bye";
private static BlockingCollection<Message> messages = new BlockingCollection<Message>();
private static void Producer()
{
int ctr = 1;
while (true)
{
messages.Add(new Message { Id = ctr, Name = $"Name-{ctr}" });
Thread.Sleep(1000);
ctr++;
}
}
private static void Consumer()
{
foreach (var message in messages.GetConsumingEnumerable())
{
Console.WriteLine(JsonConvert.SerializeObject(message));
string json = JsonConvert.SerializeObject(message);
int maxchar = 102400;
if (last_path != "")
{
long length = new FileInfo(last_path).Length;
if (length < maxchar)
{
maxchar = maxchar - unchecked((int)length);
dividefile(last_path, maxchar, json);
}
else
{
dividefile("", maxchar, json);
}
}
else
{
dividefile("", maxchar, json);
}
}
}
public static void dividefile(string path, int maxchar, string message)
{
//FileStream fileStream = new FileStream(yourfile, FileMode.Open, FileAccess.Read);
byte[] byteArray = Encoding.UTF8.GetBytes(message);
MemoryStream stream = new MemoryStream(byteArray);
using (StreamReader streamReader = new StreamReader(stream))
{
Int64 x1 = stream.Length;
char[] fileContents = new char[maxchar];
int charsRead = streamReader.Read(fileContents, 0, maxchar);
// Can't do much with 0 bytes
if (charsRead == 0)
throw new Exception("File is 0 bytes");
while (charsRead > 0)
{
x1 = x1 - maxchar;
if (x1 > 0)
{
string s = new string(fileContents);
if (path == "")
{
last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
path = "";
}
else
{
last_path = path;
}
AppendTransaction(last_path, s);
charsRead = streamReader.Read(fileContents, 0, maxchar);
}
else
{
int m = (int)(((x1 + maxchar) % maxchar));
string messagechunk = new string(fileContents, 0, m);
if (path == "")
{
last_path = Path.Combine(@"C:\TEMP", $"File-{ DateTime.Now.ToString("yyyyMMddHHmmssfff")}.json");
path = "";
}
else
{
last_path = path;
}
AppendTransaction(last_path, messagechunk);
charsRead = streamReader.Read(fileContents, 0, m);
}
}
}
}
private static void AppendTransaction(string path , string transaction)
{
string filename = path;
bool firstTransaction = !File.Exists(filename);
JsonSerializer ser = new JsonSerializer();
ser.Formatting = Formatting.Indented;
ser.TypeNameHandling = TypeNameHandling.Auto;
using (var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
Encoding enc = firstTransaction ? new UTF8Encoding(true) : new UTF8Encoding(false);
using (var sw = new StreamWriter(fs, enc))
using (var jtw = new JsonTextWriter(sw))
{
if (firstTransaction)
{
sw.Write("[");
sw.Flush();
}
else
{
fs.Seek(-Encoding.UTF8.GetByteCount("]"), SeekOrigin.End);
sw.Write(",");
sw.Flush();
}
ser.Serialize(jtw, transaction);
sw.Write(']');
}
}
}
static void Main(string[] args)
{
var producer = Task.Factory.StartNew(() => Producer());
var consumer = Task.Factory.StartNew(() => Consumer());
Console.Read();
}
class Message
{
public int ProductorThreadID { get; set; }
public int CustomerThreadID { get; set; }
public string key { get; set; }
public string content { get; set; }
public string Name { get; internal set; }
public int Id { get; internal set; }
public bool endThread()
{
return string.Compare(key, Program.BYE) == 0;
}
public string ToString(bool isProductor)
{
return string.Format("{0} Thread ID {1} : {2}", isProductor ? "Productor" : "Customer",
isProductor ? ProductorThreadID.ToString() : CustomerThreadID.ToString(),
content);
}
}
}
}