C# serialize huge data more than 100M via splitting into batch and concatenating as one big json file

Install-Package MessagePack
Install-Package Newtonsoft.json

using MessagePack;
using Newtonsoft.Json;
using System.CodeDom.Compiler;
using System.Collections;
using System.ComponentModel.DataAnnotations;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Formatting = Newtonsoft.Json.Formatting;
using KeyAttribute = MessagePack.KeyAttribute;

namespace ConsoleApp11
{
    internal class Program
    {
        static int idx = 0;
        static void Main(string[] args)
        {
            //ReadBinFile();
            //WriteToBinFile();
            SplitByBatchSize(5, 190000000, 500000);
            //JsonDeserialize("Json_202603271009288890.json");
            Console.WriteLine($"{DateTime.Now.ToString("O")},finished");
        }

        static void JsonDeserialize(string jsonFile)
        {
            if (!File.Exists(jsonFile))
            {
                Console.WriteLine($"{DateTime.Now},json file:{jsonFile} does not exist!");
                return;
            }
            using (StreamReader reader = new StreamReader(jsonFile, Encoding.UTF8, false))
            {
                string jsonStr = reader.ReadToEnd();
                var dataList = JsonConvert.DeserializeObject<List<Book>>(jsonStr);
                if (dataList != null && dataList.Any())
                {
                    Console.WriteLine($"{DateTime.Now.ToString("O")},len:{dataList.Count}");
                }
            }
        }

        static int GetIncrementIdx()
        {
            return Interlocked.Increment(ref idx);
        }

        static void ReadBinFile(string binFile = "Bin_202603271118076128.bin")
        {
            byte[] readBytes = File.ReadAllBytes(binFile);
            List<Book> list = MessagePackSerializer.Deserialize<List<Book>>(readBytes);
            Console.WriteLine($"{DateTime.Now},len:{list.Count}");
        }

        private static void WriteToBinFile()
        {
            string binFile = $"Bin_{DateTime.Now.ToString("yyyyMMddHHmmssffff")}.bin";
            var booksList = GenerateDataList(0, 10000000);
            SaveDataListInBinary(booksList, binFile);
            Console.WriteLine($"{DateTime.Now},save in {binFile}");
        }

        private static void SaveDataListInBinary(List<Book> dataList, string binFile = "")
        {
            byte[] bytes = MessagePackSerializer.Serialize(dataList);
            File.WriteAllBytes(binFile, bytes);
        }

        private static void SplitByBatchSize(int start = 0, int end = 0, int batch_size = 0)
        {
            if (start >= end)
            {
                Console.WriteLine($"{DateTime.Now.ToString("O")},start:{start}>=end:{end}");
                return;
            }

            int batch_count = (end - start + batch_size + 1) / batch_size;
            Console.WriteLine($"{DateTime.Now.ToString("O")},start:{start},end:{end},batch size:{batch_size},batch_count:{batch_count}");

            var batch_list = Enumerable.Range(0, batch_count).ToList();
            string jsonFile = $"Json_{DateTime.Now.ToString("yyyyMMddHHmmssffff")}.json";
            foreach (var batch in batch_list)
            {
                int startIdx = start + batch * batch_size;
                int endIdx = Math.Min(start + batch * batch_size + batch_size, end);
                var booksList = GenerateDataList(startIdx, endIdx);
                if (booksList != null && booksList.Any())
                {
                    AppendDataListToJson(booksList, jsonFile, startIdx, endIdx, startIdx == start, endIdx == end, batch);
                }
            }

            Console.WriteLine($"{DateTime.Now},start:{start},end:{end},jsonFile:{jsonFile} finished");
        }

        private static void AppendDataListToJson(List<Book> booksList, string jsonFile, int startIdx, int endIdx, bool isStarted, bool isEnded, int batch)
        {
            string jsonStr = JsonConvert.SerializeObject(booksList, Formatting.Indented);
            jsonStr = jsonStr.Trim('[', ']').Trim();
            using (StreamWriter jsonWriter = new StreamWriter(jsonFile, true, System.Text.Encoding.UTF8))
            {
                if (isStarted)
                {
                    jsonWriter.Write("[\n  ");
                    jsonWriter.Write(jsonStr);
                }
                else if (isEnded)
                {
                    jsonWriter.Write(",\n  ");
                    jsonWriter.Write(jsonStr);
                    jsonWriter.Write("\n]");
                }
                else
                {
                    jsonWriter.Write(",\n  ");
                    jsonWriter.Write(jsonStr);
                }
            }
            Console.WriteLine($"{DateTime.Now},startIdx:{startIdx},endIdx:{endIdx},batch:{batch + 1}");
        }

        private static List<Book> GenerateDataList(int startIdx, int endIdx)
        {
            idx = startIdx - 1;
            List<Book> booksList = new List<Book>();
            if (startIdx >= endIdx)
            {
                Console.WriteLine($"{DateTime.Now},startIdx>=endIdx");
                return booksList;
            }

            for (int i = startIdx; i <= endIdx; i++)
            {
                var a = GetIncrementIdx();
                booksList.Add(new Book()
                {
                    Id = a,
                    Name = $"Name_{a}",
                    ISBN = $"ISBN_{a}_{Guid.NewGuid():N}",
                    Author = $"Author_{a}",
                    Abstract = $"Abstract_{a}",
                    Summary = $"Summary_{a}",
                    Title = $"Title_{a}",
                    Topic = $"Topic_{a}"
                });
            }
            return booksList;
        }
    }

    [MessagePackObject]
    class Book
    {
        [Key(0)] public int Id { get; set; }
        [Key(1)] public string Name { get; set; }
        [Key(2)] public string Author { get; set; }
        [Key(3)] public string Abstract { get; set; }
        [Key(4)] public string ISBN { get; set; }
        [Key(5)] public string Summary { get; set; }
        [Key(6)] public string Title { get; set; }
        [Key(7)] public string Topic { get; set; }
    }
}

2026-03-29T22:32:03.9580593+08:00,start:5,end:190000000,batch size:500000,batch_count:380
2026-03-29 22:32:06,startIdx:5,endIdx:500005,batch:1
2026-03-29 22:32:07,startIdx:500005,endIdx:1000005,batch:2
2026-03-29 22:32:08,startIdx:1000005,endIdx:1500005,batch:3
2026-03-29 22:32:09,startIdx:1500005,endIdx:2000005,batch:4
2026-03-29 22:32:10,startIdx:2000005,endIdx:2500005,batch:5
2026-03-29 22:32:11,startIdx:2500005,endIdx:3000005,batch:6
2026-03-29 22:32:12,startIdx:3000005,endIdx:3500005,batch:7
2026-03-29 22:32:13,startIdx:3500005,endIdx:4000005,batch:8
2026-03-29 22:32:14,startIdx:4000005,endIdx:4500005,batch:9
2026-03-29 22:32:15,startIdx:4500005,endIdx:5000005,batch:10
2026-03-29 22:32:16,startIdx:5000005,endIdx:5500005,batch:11
2026-03-29 22:32:17,startIdx:5500005,endIdx:6000005,batch:12
2026-03-29 22:32:18,startIdx:6000005,endIdx:6500005,batch:13
2026-03-29 22:32:19,startIdx:6500005,endIdx:7000005,batch:14
2026-03-29 22:32:20,startIdx:7000005,endIdx:7500005,batch:15
2026-03-29 22:32:21,startIdx:7500005,endIdx:8000005,batch:16
2026-03-29 22:32:22,startIdx:8000005,endIdx:8500005,batch:17
2026-03-29 22:32:23,startIdx:8500005,endIdx:9000005,batch:18
2026-03-29 22:32:24,startIdx:9000005,endIdx:9500005,batch:19
2026-03-29 22:32:25,startIdx:9500005,endIdx:10000005,batch:20
2026-03-29 22:32:26,startIdx:10000005,endIdx:10500005,batch:21
2026-03-29 22:32:27,startIdx:10500005,endIdx:11000005,batch:22
2026-03-29 22:32:28,startIdx:11000005,endIdx:11500005,batch:23
2026-03-29 22:32:29,startIdx:11500005,endIdx:12000005,batch:24
2026-03-29 22:32:30,startIdx:12000005,endIdx:12500005,batch:25
2026-03-29 22:32:31,startIdx:12500005,endIdx:13000005,batch:26
2026-03-29 22:32:32,startIdx:13000005,endIdx:13500005,batch:27
2026-03-29 22:32:33,startIdx:13500005,endIdx:14000005,batch:28
2026-03-29 22:32:34,startIdx:14000005,endIdx:14500005,batch:29
2026-03-29 22:32:35,startIdx:14500005,endIdx:15000005,batch:30
2026-03-29 22:32:36,startIdx:15000005,endIdx:15500005,batch:31
2026-03-29 22:32:38,startIdx:15500005,endIdx:16000005,batch:32
2026-03-29 22:32:39,startIdx:16000005,endIdx:16500005,batch:33
2026-03-29 22:32:39,startIdx:16500005,endIdx:17000005,batch:34
2026-03-29 22:32:41,startIdx:17000005,endIdx:17500005,batch:35
2026-03-29 22:32:42,startIdx:17500005,endIdx:18000005,batch:36
2026-03-29 22:32:43,startIdx:18000005,endIdx:18500005,batch:37
2026-03-29 22:32:44,startIdx:18500005,endIdx:19000005,batch:38
2026-03-29 22:32:45,startIdx:19000005,endIdx:19500005,batch:39
2026-03-29 22:32:46,startIdx:19500005,endIdx:20000005,batch:40
2026-03-29 22:32:47,startIdx:20000005,endIdx:20500005,batch:41
2026-03-29 22:32:48,startIdx:20500005,endIdx:21000005,batch:42
2026-03-29 22:32:49,startIdx:21000005,endIdx:21500005,batch:43
2026-03-29 22:32:50,startIdx:21500005,endIdx:22000005,batch:44
2026-03-29 22:32:51,startIdx:22000005,endIdx:22500005,batch:45
2026-03-29 22:32:52,startIdx:22500005,endIdx:23000005,batch:46
2026-03-29 22:32:53,startIdx:23000005,endIdx:23500005,batch:47
2026-03-29 22:32:54,startIdx:23500005,endIdx:24000005,batch:48
2026-03-29 22:32:55,startIdx:24000005,endIdx:24500005,batch:49
2026-03-29 22:32:56,startIdx:24500005,endIdx:25000005,batch:50
2026-03-29 22:32:57,startIdx:25000005,endIdx:25500005,batch:51
2026-03-29 22:32:58,startIdx:25500005,endIdx:26000005,batch:52
2026-03-29 22:32:59,startIdx:26000005,endIdx:26500005,batch:53
2026-03-29 22:33:00,startIdx:26500005,endIdx:27000005,batch:54
2026-03-29 22:33:01,startIdx:27000005,endIdx:27500005,batch:55
2026-03-29 22:33:02,startIdx:27500005,endIdx:28000005,batch:56
2026-03-29 22:33:03,startIdx:28000005,endIdx:28500005,batch:57
2026-03-29 22:33:04,startIdx:28500005,endIdx:29000005,batch:58
2026-03-29 22:33:05,startIdx:29000005,endIdx:29500005,batch:59
2026-03-29 22:33:06,startIdx:29500005,endIdx:30000005,batch:60
2026-03-29 22:33:07,startIdx:30000005,endIdx:30500005,batch:61
2026-03-29 22:33:08,startIdx:30500005,endIdx:31000005,batch:62
2026-03-29 22:33:09,startIdx:31000005,endIdx:31500005,batch:63
2026-03-29 22:33:10,startIdx:31500005,endIdx:32000005,batch:64
2026-03-29 22:33:11,startIdx:32000005,endIdx:32500005,batch:65
2026-03-29 22:33:12,startIdx:32500005,endIdx:33000005,batch:66
2026-03-29 22:33:13,startIdx:33000005,endIdx:33500005,batch:67
2026-03-29 22:33:14,startIdx:33500005,endIdx:34000005,batch:68
2026-03-29 22:33:15,startIdx:34000005,endIdx:34500005,batch:69
2026-03-29 22:33:16,startIdx:34500005,endIdx:35000005,batch:70
2026-03-29 22:33:17,startIdx:35000005,endIdx:35500005,batch:71
2026-03-29 22:33:18,startIdx:35500005,endIdx:36000005,batch:72
2026-03-29 22:33:19,startIdx:36000005,endIdx:36500005,batch:73
2026-03-29 22:33:20,startIdx:36500005,endIdx:37000005,batch:74
2026-03-29 22:33:21,startIdx:37000005,endIdx:37500005,batch:75
2026-03-29 22:33:22,startIdx:37500005,endIdx:38000005,batch:76
2026-03-29 22:33:24,startIdx:38000005,endIdx:38500005,batch:77
2026-03-29 22:33:24,startIdx:38500005,endIdx:39000005,batch:78
2026-03-29 22:33:25,startIdx:39000005,endIdx:39500005,batch:79
2026-03-29 22:33:27,startIdx:39500005,endIdx:40000005,batch:80
2026-03-29 22:33:28,startIdx:40000005,endIdx:40500005,batch:81
2026-03-29 22:33:29,startIdx:40500005,endIdx:41000005,batch:82
2026-03-29 22:33:29,startIdx:41000005,endIdx:41500005,batch:83
2026-03-29 22:33:31,startIdx:41500005,endIdx:42000005,batch:84
2026-03-29 22:33:32,startIdx:42000005,endIdx:42500005,batch:85
2026-03-29 22:33:33,startIdx:42500005,endIdx:43000005,batch:86
2026-03-29 22:33:34,startIdx:43000005,endIdx:43500005,batch:87
2026-03-29 22:33:35,startIdx:43500005,endIdx:44000005,batch:88
2026-03-29 22:33:36,startIdx:44000005,endIdx:44500005,batch:89
2026-03-29 22:33:37,startIdx:44500005,endIdx:45000005,batch:90
2026-03-29 22:33:38,startIdx:45000005,endIdx:45500005,batch:91
2026-03-29 22:33:39,startIdx:45500005,endIdx:46000005,batch:92
2026-03-29 22:33:40,startIdx:46000005,endIdx:46500005,batch:93
2026-03-29 22:33:41,startIdx:46500005,endIdx:47000005,batch:94
2026-03-29 22:33:42,startIdx:47000005,endIdx:47500005,batch:95
2026-03-29 22:33:43,startIdx:47500005,endIdx:48000005,batch:96
2026-03-29 22:33:44,startIdx:48000005,endIdx:48500005,batch:97
2026-03-29 22:33:45,startIdx:48500005,endIdx:49000005,batch:98
2026-03-29 22:33:46,startIdx:49000005,endIdx:49500005,batch:99
2026-03-29 22:33:47,startIdx:49500005,endIdx:50000005,batch:100
2026-03-29 22:33:48,startIdx:50000005,endIdx:50500005,batch:101
2026-03-29 22:33:49,startIdx:50500005,endIdx:51000005,batch:102
2026-03-29 22:33:50,startIdx:51000005,endIdx:51500005,batch:103
2026-03-29 22:33:51,startIdx:51500005,endIdx:52000005,batch:104
2026-03-29 22:33:52,startIdx:52000005,endIdx:52500005,batch:105
2026-03-29 22:33:53,startIdx:52500005,endIdx:53000005,batch:106
2026-03-29 22:33:54,startIdx:53000005,endIdx:53500005,batch:107
2026-03-29 22:33:55,startIdx:53500005,endIdx:54000005,batch:108
2026-03-29 22:33:56,startIdx:54000005,endIdx:54500005,batch:109
2026-03-29 22:33:57,startIdx:54500005,endIdx:55000005,batch:110
2026-03-29 22:33:58,startIdx:55000005,endIdx:55500005,batch:111
2026-03-29 22:33:59,startIdx:55500005,endIdx:56000005,batch:112
2026-03-29 22:34:00,startIdx:56000005,endIdx:56500005,batch:113
2026-03-29 22:34:01,startIdx:56500005,endIdx:57000005,batch:114
2026-03-29 22:34:02,startIdx:57000005,endIdx:57500005,batch:115
2026-03-29 22:34:04,startIdx:57500005,endIdx:58000005,batch:116
2026-03-29 22:34:05,startIdx:58000005,endIdx:58500005,batch:117
2026-03-29 22:34:06,startIdx:58500005,endIdx:59000005,batch:118
2026-03-29 22:34:07,startIdx:59000005,endIdx:59500005,batch:119
2026-03-29 22:34:08,startIdx:59500005,endIdx:60000005,batch:120
2026-03-29 22:34:09,startIdx:60000005,endIdx:60500005,batch:121
2026-03-29 22:34:10,startIdx:60500005,endIdx:61000005,batch:122
2026-03-29 22:34:11,startIdx:61000005,endIdx:61500005,batch:123
2026-03-29 22:34:12,startIdx:61500005,endIdx:62000005,batch:124
2026-03-29 22:34:13,startIdx:62000005,endIdx:62500005,batch:125
2026-03-29 22:34:14,startIdx:62500005,endIdx:63000005,batch:126
2026-03-29 22:34:15,startIdx:63000005,endIdx:63500005,batch:127
2026-03-29 22:34:16,startIdx:63500005,endIdx:64000005,batch:128
2026-03-29 22:34:17,startIdx:64000005,endIdx:64500005,batch:129
2026-03-29 22:34:18,startIdx:64500005,endIdx:65000005,batch:130
2026-03-29 22:34:19,startIdx:65000005,endIdx:65500005,batch:131
2026-03-29 22:34:20,startIdx:65500005,endIdx:66000005,batch:132
2026-03-29 22:34:21,startIdx:66000005,endIdx:66500005,batch:133
2026-03-29 22:34:22,startIdx:66500005,endIdx:67000005,batch:134
2026-03-29 22:34:24,startIdx:67000005,endIdx:67500005,batch:135
2026-03-29 22:34:24,startIdx:67500005,endIdx:68000005,batch:136
2026-03-29 22:34:26,startIdx:68000005,endIdx:68500005,batch:137
2026-03-29 22:34:27,startIdx:68500005,endIdx:69000005,batch:138
2026-03-29 22:34:28,startIdx:69000005,endIdx:69500005,batch:139
2026-03-29 22:34:29,startIdx:69500005,endIdx:70000005,batch:140
2026-03-29 22:34:30,startIdx:70000005,endIdx:70500005,batch:141
2026-03-29 22:34:31,startIdx:70500005,endIdx:71000005,batch:142
2026-03-29 22:34:32,startIdx:71000005,endIdx:71500005,batch:143
2026-03-29 22:34:33,startIdx:71500005,endIdx:72000005,batch:144
2026-03-29 22:34:34,startIdx:72000005,endIdx:72500005,batch:145
2026-03-29 22:34:35,startIdx:72500005,endIdx:73000005,batch:146
2026-03-29 22:34:36,startIdx:73000005,endIdx:73500005,batch:147
2026-03-29 22:34:37,startIdx:73500005,endIdx:74000005,batch:148
2026-03-29 22:34:38,startIdx:74000005,endIdx:74500005,batch:149
2026-03-29 22:34:39,startIdx:74500005,endIdx:75000005,batch:150
2026-03-29 22:34:41,startIdx:75000005,endIdx:75500005,batch:151
2026-03-29 22:34:43,startIdx:75500005,endIdx:76000005,batch:152
2026-03-29 22:34:46,startIdx:76000005,endIdx:76500005,batch:153
2026-03-29 22:34:49,startIdx:76500005,endIdx:77000005,batch:154
2026-03-29 22:34:51,startIdx:77000005,endIdx:77500005,batch:155
2026-03-29 22:34:53,startIdx:77500005,endIdx:78000005,batch:156
2026-03-29 22:34:56,startIdx:78000005,endIdx:78500005,batch:157
2026-03-29 22:34:58,startIdx:78500005,endIdx:79000005,batch:158
2026-03-29 22:35:01,startIdx:79000005,endIdx:79500005,batch:159
2026-03-29 22:35:04,startIdx:79500005,endIdx:80000005,batch:160
2026-03-29 22:35:06,startIdx:80000005,endIdx:80500005,batch:161
2026-03-29 22:35:09,startIdx:80500005,endIdx:81000005,batch:162
2026-03-29 22:35:11,startIdx:81000005,endIdx:81500005,batch:163
2026-03-29 22:35:13,startIdx:81500005,endIdx:82000005,batch:164
2026-03-29 22:35:16,startIdx:82000005,endIdx:82500005,batch:165
2026-03-29 22:35:19,startIdx:82500005,endIdx:83000005,batch:166
2026-03-29 22:35:22,startIdx:83000005,endIdx:83500005,batch:167
2026-03-29 22:35:24,startIdx:83500005,endIdx:84000005,batch:168
2026-03-29 22:35:27,startIdx:84000005,endIdx:84500005,batch:169
2026-03-29 22:35:29,startIdx:84500005,endIdx:85000005,batch:170
2026-03-29 22:35:32,startIdx:85000005,endIdx:85500005,batch:171
2026-03-29 22:35:34,startIdx:85500005,endIdx:86000005,batch:172
2026-03-29 22:35:37,startIdx:86000005,endIdx:86500005,batch:173
2026-03-29 22:35:39,startIdx:86500005,endIdx:87000005,batch:174
2026-03-29 22:35:42,startIdx:87000005,endIdx:87500005,batch:175
2026-03-29 22:35:44,startIdx:87500005,endIdx:88000005,batch:176
2026-03-29 22:35:46,startIdx:88000005,endIdx:88500005,batch:177
2026-03-29 22:35:49,startIdx:88500005,endIdx:89000005,batch:178
2026-03-29 22:35:51,startIdx:89000005,endIdx:89500005,batch:179
2026-03-29 22:35:54,startIdx:89500005,endIdx:90000005,batch:180
2026-03-29 22:35:57,startIdx:90000005,endIdx:90500005,batch:181
2026-03-29 22:35:59,startIdx:90500005,endIdx:91000005,batch:182
2026-03-29 22:36:02,startIdx:91000005,endIdx:91500005,batch:183
2026-03-29 22:36:04,startIdx:91500005,endIdx:92000005,batch:184
2026-03-29 22:36:07,startIdx:92000005,endIdx:92500005,batch:185
2026-03-29 22:36:09,startIdx:92500005,endIdx:93000005,batch:186
2026-03-29 22:36:12,startIdx:93000005,endIdx:93500005,batch:187
2026-03-29 22:36:14,startIdx:93500005,endIdx:94000005,batch:188
2026-03-29 22:36:17,startIdx:94000005,endIdx:94500005,batch:189
2026-03-29 22:36:19,startIdx:94500005,endIdx:95000005,batch:190
2026-03-29 22:36:22,startIdx:95000005,endIdx:95500005,batch:191
2026-03-29 22:36:24,startIdx:95500005,endIdx:96000005,batch:192
2026-03-29 22:36:26,startIdx:96000005,endIdx:96500005,batch:193
2026-03-29 22:36:27,startIdx:96500005,endIdx:97000005,batch:194
2026-03-29 22:36:28,startIdx:97000005,endIdx:97500005,batch:195
2026-03-29 22:36:29,startIdx:97500005,endIdx:98000005,batch:196
2026-03-29 22:36:30,startIdx:98000005,endIdx:98500005,batch:197
2026-03-29 22:36:32,startIdx:98500005,endIdx:99000005,batch:198
2026-03-29 22:36:34,startIdx:99000005,endIdx:99500005,batch:199
2026-03-29 22:36:36,startIdx:99500005,endIdx:100000005,batch:200
2026-03-29 22:36:39,startIdx:100000005,endIdx:100500005,batch:201
2026-03-29 22:36:42,startIdx:100500005,endIdx:101000005,batch:202
2026-03-29 22:36:44,startIdx:101000005,endIdx:101500005,batch:203
2026-03-29 22:36:46,startIdx:101500005,endIdx:102000005,batch:204
2026-03-29 22:36:48,startIdx:102000005,endIdx:102500005,batch:205
2026-03-29 22:36:51,startIdx:102500005,endIdx:103000005,batch:206
2026-03-29 22:36:53,startIdx:103000005,endIdx:103500005,batch:207
2026-03-29 22:36:55,startIdx:103500005,endIdx:104000005,batch:208
2026-03-29 22:36:58,startIdx:104000005,endIdx:104500005,batch:209
2026-03-29 22:37:00,startIdx:104500005,endIdx:105000005,batch:210
2026-03-29 22:37:03,startIdx:105000005,endIdx:105500005,batch:211
2026-03-29 22:37:05,startIdx:105500005,endIdx:106000005,batch:212
2026-03-29 22:37:07,startIdx:106000005,endIdx:106500005,batch:213
2026-03-29 22:37:10,startIdx:106500005,endIdx:107000005,batch:214
2026-03-29 22:37:12,startIdx:107000005,endIdx:107500005,batch:215
2026-03-29 22:37:14,startIdx:107500005,endIdx:108000005,batch:216
2026-03-29 22:37:17,startIdx:108000005,endIdx:108500005,batch:217
2026-03-29 22:37:19,startIdx:108500005,endIdx:109000005,batch:218
2026-03-29 22:37:21,startIdx:109000005,endIdx:109500005,batch:219
2026-03-29 22:37:24,startIdx:109500005,endIdx:110000005,batch:220
2026-03-29 22:37:26,startIdx:110000005,endIdx:110500005,batch:221
2026-03-29 22:37:29,startIdx:110500005,endIdx:111000005,batch:222
2026-03-29 22:37:31,startIdx:111000005,endIdx:111500005,batch:223
2026-03-29 22:37:33,startIdx:111500005,endIdx:112000005,batch:224
2026-03-29 22:37:35,startIdx:112000005,endIdx:112500005,batch:225
2026-03-29 22:37:38,startIdx:112500005,endIdx:113000005,batch:226
2026-03-29 22:37:40,startIdx:113000005,endIdx:113500005,batch:227
2026-03-29 22:37:43,startIdx:113500005,endIdx:114000005,batch:228
2026-03-29 22:37:45,startIdx:114000005,endIdx:114500005,batch:229
2026-03-29 22:37:47,startIdx:114500005,endIdx:115000005,batch:230
2026-03-29 22:37:49,startIdx:115000005,endIdx:115500005,batch:231
2026-03-29 22:37:52,startIdx:115500005,endIdx:116000005,batch:232
2026-03-29 22:37:54,startIdx:116000005,endIdx:116500005,batch:233
2026-03-29 22:37:57,startIdx:116500005,endIdx:117000005,batch:234
2026-03-29 22:38:00,startIdx:117000005,endIdx:117500005,batch:235
2026-03-29 22:38:02,startIdx:117500005,endIdx:118000005,batch:236
2026-03-29 22:38:04,startIdx:118000005,endIdx:118500005,batch:237
2026-03-29 22:38:06,startIdx:118500005,endIdx:119000005,batch:238
2026-03-29 22:38:07,startIdx:119000005,endIdx:119500005,batch:239
2026-03-29 22:38:08,startIdx:119500005,endIdx:120000005,batch:240
2026-03-29 22:38:09,startIdx:120000005,endIdx:120500005,batch:241
2026-03-29 22:38:10,startIdx:120500005,endIdx:121000005,batch:242
2026-03-29 22:38:11,startIdx:121000005,endIdx:121500005,batch:243
2026-03-29 22:38:12,startIdx:121500005,endIdx:122000005,batch:244
2026-03-29 22:38:14,startIdx:122000005,endIdx:122500005,batch:245
2026-03-29 22:38:15,startIdx:122500005,endIdx:123000005,batch:246
2026-03-29 22:38:16,startIdx:123000005,endIdx:123500005,batch:247
2026-03-29 22:38:17,startIdx:123500005,endIdx:124000005,batch:248
2026-03-29 22:38:19,startIdx:124000005,endIdx:124500005,batch:249
2026-03-29 22:38:20,startIdx:124500005,endIdx:125000005,batch:250
2026-03-29 22:38:21,startIdx:125000005,endIdx:125500005,batch:251
2026-03-29 22:38:22,startIdx:125500005,endIdx:126000005,batch:252
2026-03-29 22:38:23,startIdx:126000005,endIdx:126500005,batch:253
2026-03-29 22:38:24,startIdx:126500005,endIdx:127000005,batch:254
2026-03-29 22:38:25,startIdx:127000005,endIdx:127500005,batch:255
2026-03-29 22:38:26,startIdx:127500005,endIdx:128000005,batch:256
2026-03-29 22:38:28,startIdx:128000005,endIdx:128500005,batch:257
2026-03-29 22:38:29,startIdx:128500005,endIdx:129000005,batch:258
2026-03-29 22:38:30,startIdx:129000005,endIdx:129500005,batch:259
2026-03-29 22:38:31,startIdx:129500005,endIdx:130000005,batch:260
2026-03-29 22:38:32,startIdx:130000005,endIdx:130500005,batch:261
2026-03-29 22:38:33,startIdx:130500005,endIdx:131000005,batch:262
2026-03-29 22:38:34,startIdx:131000005,endIdx:131500005,batch:263
2026-03-29 22:38:35,startIdx:131500005,endIdx:132000005,batch:264
2026-03-29 22:38:37,startIdx:132000005,endIdx:132500005,batch:265
2026-03-29 22:38:38,startIdx:132500005,endIdx:133000005,batch:266
2026-03-29 22:38:39,startIdx:133000005,endIdx:133500005,batch:267
2026-03-29 22:38:40,startIdx:133500005,endIdx:134000005,batch:268
2026-03-29 22:38:41,startIdx:134000005,endIdx:134500005,batch:269
2026-03-29 22:38:42,startIdx:134500005,endIdx:135000005,batch:270
2026-03-29 22:38:43,startIdx:135000005,endIdx:135500005,batch:271
2026-03-29 22:38:44,startIdx:135500005,endIdx:136000005,batch:272
2026-03-29 22:38:46,startIdx:136000005,endIdx:136500005,batch:273
2026-03-29 22:38:47,startIdx:136500005,endIdx:137000005,batch:274
2026-03-29 22:38:48,startIdx:137000005,endIdx:137500005,batch:275
2026-03-29 22:38:49,startIdx:137500005,endIdx:138000005,batch:276
2026-03-29 22:38:50,startIdx:138000005,endIdx:138500005,batch:277
2026-03-29 22:38:51,startIdx:138500005,endIdx:139000005,batch:278
2026-03-29 22:38:53,startIdx:139000005,endIdx:139500005,batch:279
2026-03-29 22:38:54,startIdx:139500005,endIdx:140000005,batch:280
2026-03-29 22:38:55,startIdx:140000005,endIdx:140500005,batch:281
2026-03-29 22:38:56,startIdx:140500005,endIdx:141000005,batch:282
2026-03-29 22:38:57,startIdx:141000005,endIdx:141500005,batch:283
2026-03-29 22:38:58,startIdx:141500005,endIdx:142000005,batch:284
2026-03-29 22:38:59,startIdx:142000005,endIdx:142500005,batch:285
2026-03-29 22:39:00,startIdx:142500005,endIdx:143000005,batch:286
2026-03-29 22:39:01,startIdx:143000005,endIdx:143500005,batch:287
2026-03-29 22:39:02,startIdx:143500005,endIdx:144000005,batch:288
2026-03-29 22:39:03,startIdx:144000005,endIdx:144500005,batch:289
2026-03-29 22:39:04,startIdx:144500005,endIdx:145000005,batch:290
2026-03-29 22:39:05,startIdx:145000005,endIdx:145500005,batch:291
2026-03-29 22:39:06,startIdx:145500005,endIdx:146000005,batch:292
2026-03-29 22:39:07,startIdx:146000005,endIdx:146500005,batch:293
2026-03-29 22:39:08,startIdx:146500005,endIdx:147000005,batch:294
2026-03-29 22:39:09,startIdx:147000005,endIdx:147500005,batch:295
2026-03-29 22:39:10,startIdx:147500005,endIdx:148000005,batch:296
2026-03-29 22:39:12,startIdx:148000005,endIdx:148500005,batch:297
2026-03-29 22:39:13,startIdx:148500005,endIdx:149000005,batch:298
2026-03-29 22:39:14,startIdx:149000005,endIdx:149500005,batch:299
2026-03-29 22:39:15,startIdx:149500005,endIdx:150000005,batch:300
2026-03-29 22:39:16,startIdx:150000005,endIdx:150500005,batch:301
2026-03-29 22:39:17,startIdx:150500005,endIdx:151000005,batch:302
2026-03-29 22:39:18,startIdx:151000005,endIdx:151500005,batch:303
2026-03-29 22:39:19,startIdx:151500005,endIdx:152000005,batch:304
2026-03-29 22:39:20,startIdx:152000005,endIdx:152500005,batch:305
2026-03-29 22:39:21,startIdx:152500005,endIdx:153000005,batch:306
2026-03-29 22:39:22,startIdx:153000005,endIdx:153500005,batch:307
2026-03-29 22:39:23,startIdx:153500005,endIdx:154000005,batch:308
2026-03-29 22:39:24,startIdx:154000005,endIdx:154500005,batch:309
2026-03-29 22:39:25,startIdx:154500005,endIdx:155000005,batch:310
2026-03-29 22:39:27,startIdx:155000005,endIdx:155500005,batch:311
2026-03-29 22:39:27,startIdx:155500005,endIdx:156000005,batch:312
2026-03-29 22:39:29,startIdx:156000005,endIdx:156500005,batch:313
2026-03-29 22:39:30,startIdx:156500005,endIdx:157000005,batch:314
2026-03-29 22:39:31,startIdx:157000005,endIdx:157500005,batch:315
2026-03-29 22:39:32,startIdx:157500005,endIdx:158000005,batch:316
2026-03-29 22:39:33,startIdx:158000005,endIdx:158500005,batch:317
2026-03-29 22:39:34,startIdx:158500005,endIdx:159000005,batch:318
2026-03-29 22:39:35,startIdx:159000005,endIdx:159500005,batch:319
2026-03-29 22:39:36,startIdx:159500005,endIdx:160000005,batch:320
2026-03-29 22:39:37,startIdx:160000005,endIdx:160500005,batch:321
2026-03-29 22:39:38,startIdx:160500005,endIdx:161000005,batch:322
2026-03-29 22:39:39,startIdx:161000005,endIdx:161500005,batch:323
2026-03-29 22:39:40,startIdx:161500005,endIdx:162000005,batch:324
2026-03-29 22:39:41,startIdx:162000005,endIdx:162500005,batch:325
2026-03-29 22:39:42,startIdx:162500005,endIdx:163000005,batch:326
2026-03-29 22:39:44,startIdx:163000005,endIdx:163500005,batch:327
2026-03-29 22:39:45,startIdx:163500005,endIdx:164000005,batch:328
2026-03-29 22:39:46,startIdx:164000005,endIdx:164500005,batch:329
2026-03-29 22:39:47,startIdx:164500005,endIdx:165000005,batch:330
2026-03-29 22:39:48,startIdx:165000005,endIdx:165500005,batch:331
2026-03-29 22:39:49,startIdx:165500005,endIdx:166000005,batch:332
2026-03-29 22:39:50,startIdx:166000005,endIdx:166500005,batch:333
2026-03-29 22:39:51,startIdx:166500005,endIdx:167000005,batch:334
2026-03-29 22:39:52,startIdx:167000005,endIdx:167500005,batch:335
2026-03-29 22:39:53,startIdx:167500005,endIdx:168000005,batch:336
2026-03-29 22:39:54,startIdx:168000005,endIdx:168500005,batch:337
2026-03-29 22:39:55,startIdx:168500005,endIdx:169000005,batch:338
2026-03-29 22:39:57,startIdx:169000005,endIdx:169500005,batch:339
2026-03-29 22:39:58,startIdx:169500005,endIdx:170000005,batch:340
2026-03-29 22:39:59,startIdx:170000005,endIdx:170500005,batch:341
2026-03-29 22:40:00,startIdx:170500005,endIdx:171000005,batch:342
2026-03-29 22:40:01,startIdx:171000005,endIdx:171500005,batch:343
2026-03-29 22:40:02,startIdx:171500005,endIdx:172000005,batch:344
2026-03-29 22:40:03,startIdx:172000005,endIdx:172500005,batch:345
2026-03-29 22:40:04,startIdx:172500005,endIdx:173000005,batch:346
2026-03-29 22:40:05,startIdx:173000005,endIdx:173500005,batch:347
2026-03-29 22:40:06,startIdx:173500005,endIdx:174000005,batch:348
2026-03-29 22:40:07,startIdx:174000005,endIdx:174500005,batch:349
2026-03-29 22:40:08,startIdx:174500005,endIdx:175000005,batch:350
2026-03-29 22:40:09,startIdx:175000005,endIdx:175500005,batch:351
2026-03-29 22:40:10,startIdx:175500005,endIdx:176000005,batch:352
2026-03-29 22:40:12,startIdx:176000005,endIdx:176500005,batch:353
2026-03-29 22:40:13,startIdx:176500005,endIdx:177000005,batch:354
2026-03-29 22:40:14,startIdx:177000005,endIdx:177500005,batch:355
2026-03-29 22:40:15,startIdx:177500005,endIdx:178000005,batch:356
2026-03-29 22:40:16,startIdx:178000005,endIdx:178500005,batch:357
2026-03-29 22:40:17,startIdx:178500005,endIdx:179000005,batch:358
2026-03-29 22:40:18,startIdx:179000005,endIdx:179500005,batch:359
2026-03-29 22:40:19,startIdx:179500005,endIdx:180000005,batch:360
2026-03-29 22:40:20,startIdx:180000005,endIdx:180500005,batch:361
2026-03-29 22:40:21,startIdx:180500005,endIdx:181000005,batch:362
2026-03-29 22:40:22,startIdx:181000005,endIdx:181500005,batch:363
2026-03-29 22:40:23,startIdx:181500005,endIdx:182000005,batch:364
2026-03-29 22:40:24,startIdx:182000005,endIdx:182500005,batch:365
2026-03-29 22:40:25,startIdx:182500005,endIdx:183000005,batch:366
2026-03-29 22:40:26,startIdx:183000005,endIdx:183500005,batch:367
2026-03-29 22:40:27,startIdx:183500005,endIdx:184000005,batch:368
2026-03-29 22:40:28,startIdx:184000005,endIdx:184500005,batch:369
2026-03-29 22:40:29,startIdx:184500005,endIdx:185000005,batch:370
2026-03-29 22:40:30,startIdx:185000005,endIdx:185500005,batch:371
2026-03-29 22:40:31,startIdx:185500005,endIdx:186000005,batch:372
2026-03-29 22:40:32,startIdx:186000005,endIdx:186500005,batch:373
2026-03-29 22:40:34,startIdx:186500005,endIdx:187000005,batch:374
2026-03-29 22:40:35,startIdx:187000005,endIdx:187500005,batch:375
2026-03-29 22:40:36,startIdx:187500005,endIdx:188000005,batch:376
2026-03-29 22:40:37,startIdx:188000005,endIdx:188500005,batch:377
2026-03-29 22:40:38,startIdx:188500005,endIdx:189000005,batch:378
2026-03-29 22:40:39,startIdx:189000005,endIdx:189500005,batch:379
2026-03-29 22:40:40,startIdx:189500005,endIdx:190000000,batch:380
2026-03-29 22:40:40,start:5,end:190000000,jsonFile:Json_202603292232039734.json finished
2026-03-29T22:40:40.4247208+08:00,finished

D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0\ConsoleApp11.exe (process 11724) exited with code 0 (0x0).
To automatically close the console when debugging stops, enable Tools->Options->Debugging->Automatically close the console when debugging stops.
Press any key to close this window . . .

//Powershell
Get-Content Json_202603292232039734.json -TotalCount 30

PS D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0> Get-Content Json_202603292232039734.json -TotalCount 30
[
  {
    "Id": 5,
    "Name": "Name_5",
    "Author": "Author_5",
    "Abstract": "Abstract_5",
    "ISBN": "ISBN_5_4222999a449b4da3bf356b432bce5739",
    "Summary": "Summary_5",
    "Title": "Title_5",
    "Topic": "Topic_5"
  },
  {
    "Id": 6,
    "Name": "Name_6",
    "Author": "Author_6",
    "Abstract": "Abstract_6",
    "ISBN": "ISBN_6_7a89da35e94b4317b68f9e97f6b90368",
    "Summary": "Summary_6",
    "Title": "Title_6",
    "Topic": "Topic_6"
  },
  {
    "Id": 7,
    "Name": "Name_7",
    "Author": "Author_7",
    "Abstract": "Abstract_7",
    "ISBN": "ISBN_7_5c5d5a07e655463b8d8ccd0e44d7fe1b",
    "Summary": "Summary_7",
    "Title": "Title_7",
    "Topic": "Topic_7"
PS D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0>

PS D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0> Get-Content Json_202603292232039734.json -Tail 35

PS D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0> Get-Content Json_202603292232039734.json -Tail 35
    "Summary": "Summary_189999997",
    "Title": "Title_189999997",
    "Topic": "Topic_189999997"
  },
  {
    "Id": 189999998,
    "Name": "Name_189999998",
    "Author": "Author_189999998",
    "Abstract": "Abstract_189999998",
    "ISBN": "ISBN_189999998_7040a0f7746f42aaab880a999f60e14e",
    "Summary": "Summary_189999998",
    "Title": "Title_189999998",
    "Topic": "Topic_189999998"
  },
  {
    "Id": 189999999,
    "Name": "Name_189999999",
    "Author": "Author_189999999",
    "Abstract": "Abstract_189999999",
    "ISBN": "ISBN_189999999_680b20aa26db428f9df88718b994bbe2",
    "Summary": "Summary_189999999",
    "Title": "Title_189999999",
    "Topic": "Topic_189999999"
  },
  {
    "Id": 190000000,
    "Name": "Name_190000000",
    "Author": "Author_190000000",
    "Abstract": "Abstract_190000000",
    "ISBN": "ISBN_190000000_0d223484c444471dad3d73f81d95056b",
    "Summary": "Summary_190000000",
    "Title": "Title_190000000",
    "Topic": "Topic_190000000"
  }
]
PS D:\C\ConsoleApp11\ConsoleApp11\bin\Debug\net10.0>

posted @ 2026-03-29 22:55 FredGrit 阅读(3) 评论(0) 收藏举报

刷新页面返回顶部

FredGrit

C# serialize huge data more than 100M via splitting into batch and concatenating as one big json file

公告