实现 json path 来评估函数式解析器的损耗
目的
之前就考虑评估过 函数式解析器 在dotnet这些面向对象语言上有着一些损耗，虽然表意性很强，与ABNF范式结合使用，维护性大大提升
不过由于性能考虑（以及之前认为也或许没有太多机会实现解析器），就没打算继续深究
不过现实确实有多次需要实现解析器，每次从0手写，太费时间了，比较很多实现对于解析性能并不需极致性能，(特别现在ai code疯狂的时代，老板哪会给那么多时间让你慢慢扣性能，老板们都常说ai几秒钟的事情，你个辣鸡还花几天)
所以为了血压和身体健康，只要性能将就，还是对于解析不太在意性能的场景以后就这样偷懒吧 （比如今年搞得 VKProxy 里面得动态条件和简单模板替换， 路子可行也许后面还是换掉吧，维护多累呀）
当然也有高手搞过类似的库，比如 Parlot , 不过其不支持 stream， 理想情况还是想stream 也能支持， 所以现在先简单搞一波：参考 Parlot 简单实现 string 解析器底层以及 json path 简单版
然后和 Newtonsoft.json 在简单的json path 场景比较，如果打不过，那怕没有继续的必要了
简要说明函数式解析器
这里以最简单 json path "$.Name" 举例，
我们所需要解析即为 $ 开头， . 之后字符为属性名
换成代码大致为 Parser = Char('$').And(Char('.')).And(AnyExclude("[]().,\" '\r\n@$!=<\\?&|*:")).Eof()
像 Char And 这些方法在面向对象中我们会让方法生成解析算子实例，以达到与函数式相同效果， 这也是性能损耗的一个大点
比如 Char 方法
public static Parser<char> Char(char c) => new CharLiteral(c);
具体算子实现
public class CharLiteral : Parser<char>
{
private SearchValues<char> c;
public string Value { get; private set; }
public CharLiteral(char c)
{
this.c = SearchValues.Create(new char[] { c });
Value = c.ToString();
}
public CharLiteral(string c)
{
this.c = SearchValues.Create(c);
Value = c;
}
public override bool Parse(CharParseContext context, ref ParseResult<char> result)
{
context.EnterParser(this);
var cursor = context.Cursor;
if (!cursor.Eof && c.Contains(cursor.Current))
{
var c = cursor.Current;
var start = cursor.Offset;
cursor.Advance();
result.Set(start, cursor.Offset, c);
context.ExitParser(this);
return true;
}
context.ExitParser(this);
return false;
}
}
好了，借用经典话语 这是功，这是防，这是boss 去吧
按照 [rfc9535]（https://www.rfc-editor.org/rfc/rfc9535.html） 标准， 最终 json path 解析器大致如下：
public class JsonPathParser
{
//public static Parser<char> B = Char(new char[]
//{ (char)0x20, // Space
// (char)0x09, //Horizontal tab
// (char)0x0A, // Line feed or New line
// (char)0x0D // Carriage return
//});
public static readonly Parser<char> RootIdentifier = Char('$').Name(nameof(RootIdentifier));
public static readonly Parser<int> Int = Int().Name(nameof(Int));
public static readonly Parser<char> DoubleQuoted = Char('"').Name(nameof(DoubleQuoted));
public static readonly Parser<char> SingleQuoted = Char('\'').Name(nameof(SingleQuoted));
public static readonly Parser<IStatement> WildcardSelector = Char('*').Then<IStatement>(static x => new WildcardSelectorStatment()).Name(nameof(WildcardSelector));
public static readonly Parser<IStatement> IndexSelector = Int.Then<IStatement>(static x => new IndexSelectorStatment() { Index = x }).Name(nameof(IndexSelector));
public static readonly Parser<TextSpan> StringLiteral = Between(DoubleQuoted, ZeroOrOne(Any("\"", mustHasEnd: true, escape: '\\')), DoubleQuoted).Or(Between(SingleQuoted, ZeroOrOne(Any("'", mustHasEnd: true, escape: '\\')), SingleQuoted)).Name(nameof(StringLiteral));
public static readonly Parser<IStatement> NameSelector = StringLiteral.Then<IStatement>(static x => new Member() { Name = x.Span.ToString() }).Name(nameof(NameSelector));
public static readonly Parser<int> Start = Int;
public static readonly Parser<int> End = Int;
public static readonly Parser<int> Step = Int;
public static readonly Parser<Nothing> S = IgnoreChar(new char[]
{ (char)0x20, // Space
(char)0x09, //Horizontal tab
(char)0x0A, // Line feed or New line
(char)0x0D // Carriage return
}).Name(nameof(S));
public static readonly Parser<char> CurrentNodeIdentifier = Char('@').Name(nameof(CurrentNodeIdentifier));
public static readonly Parser<char> LogicalNotOp = Char('!').Name(nameof(LogicalNotOp));
public static readonly Parser<string> ComparisonOp = Text("==").Or(Text("!=")).Or(Text("<=")).Or(Text(">=")).Or(Text("<")).Or(Text(">")).Name(nameof(ComparisonOp));
public static readonly Parser<IStatement> Num = Decimal(NumberOptions.Float).Then<IStatement>(static x => new NumberValue(x)).Name(nameof(Num));
public static readonly Parser<IStatement> True = Text("true").Then<IStatement>(static x => BoolValue.True).Name(nameof(True));
public static readonly Parser<IStatement> False = Text("false").Then<IStatement>(static x => BoolValue.False).Name(nameof(False));
public static readonly Parser<IStatement> Null = Text("null").Then<IStatement>(static x => NullValue.Value).Name(nameof(Null));
private const string name = "[]().,\" '\r\n@$!=<\\?&|*:";
//public static Parser<char> LCALPHA = Char('a', 'z');
//public static Parser<char> DIGIT = Char('0', '9');
//public static Parser<char> ALPHA = Char((char)0x41, (char)0x5A).Or(Char((char)0x61, (char)0x7A));
public static readonly Parser<IStatement> MemberNameShorthand = AnyExclude(name).Then<IStatement>(static x => new Member { Name = x.Span.ToString() }).Name(nameof(MemberNameShorthand));
//public static Parser<char> NameFirst = ALPHA.Or(Char('_')).Or(Char((char)0x80, (char)0xD7FF)).Or(Char((char)0xE000, (char)0xFFFF));
//public static Parser<char> NameChar = NameFirst.Or(DIGIT);
//public static Parser<char> FunctionNameFirst = LCALPHA;
//public static Parser<char> FunctionNameChar = FunctionNameFirst.Or(Char('_')).Or(DIGIT);
//public static Parser<string> FunctionName = FunctionNameFirst.And(ZeroOrMany(FunctionNameChar)).Then<string>(static x => throw new NotImplementedException());
public static readonly Parser<string> FunctionName = AnyExclude(name).Then<string>(static x => x.Span.ToString()).Name(nameof(FunctionName));
public static readonly Parser<IStatement> SliceSelector = Optional<int?>(Start.And(S).Then<int?>(static x => x.Item1), null).And(Char(':')).And(S).And(Optional<int?>(End.And(S).Then<int?>(static x => x.Item1), null)).And(Optional<int?>(Char(':').And(Optional<int?>(S.And(Step).Then<int?>(static x => x.Item2), null)).Then<int?>(static x => x.Item2))).Then<IStatement>(static x => new SliceStatement() { Start = x.Item1, End = x.Item4, Step = x.Item5 })
.Name(nameof(SliceSelector));
public static readonly Deferred<IStatement> LogicalExpr = Deferred<IStatement>(nameof(LogicalExpr));
public static readonly Parser<IStatement> FilterSelector = Char('?').And(S).And(LogicalExpr).Then<IStatement>(static x => new FilterSelectorStatement()
{
Statement = x.Item3
}).Name(nameof(FilterSelector));
public static readonly Parser<IStatement> Selector = NameSelector.Or(WildcardSelector).Or(SliceSelector).Or(IndexSelector).Or(FilterSelector).Name(nameof(Selector));
public static readonly Parser<IStatement> ParenExpr = Optional(LogicalNotOp.And(S)).And(Char('(')).And(S).And(LogicalExpr).And(S).And(Char(')'))
.Then<IStatement>(static x => new UnaryOperaterStatement()
{
Operator = x.Item1.Item1 == '!' ? "!" : "(",
Statement = x.Item4
}).Name(nameof(ParenExpr));
public static readonly Deferred<IReadOnlyList<(Nothing, IStatement)>> Segments = Deferred<IReadOnlyList<(Nothing, IStatement)>>(nameof(Segments));
public static readonly Deferred<IStatement> FunctionExpr = Deferred<IStatement>(nameof(FunctionExpr));
public static readonly Deferred<IStatement> JsonPathQuery = Deferred<IStatement>(nameof(JsonPathQuery));
public static readonly Parser<IStatement> RelQuery = CurrentNodeIdentifier.And(Segments).Then<IStatement>(static x => new CurrentNode() { Child = ConvertSegments(x.Item2) }).Name(nameof(RelQuery));
public static readonly Parser<IStatement> Literal = Num.Or(StringLiteral.Then<IStatement>(static x => new StringValue(x.Span.ToString()))).Or(True).Or(False).Or(Null).Name(nameof(Literal));
public static readonly Parser<IStatement> NameSegment = Char('[').And(NameSelector).And(Char(']')).Then<IStatement>(static x => x.Item2).Or(Char('.').And(MemberNameShorthand).Then<IStatement>(static x => x.Item2)).Name(nameof(NameSegment));
public static readonly Parser<IStatement> IndexSegment = Char('[').And(IndexSelector).And(Char(']')).Then<IStatement>(static x => x.Item2).Name(nameof(IndexSegment));
public static readonly Parser<IStatement> SingularQuerySegments = ZeroOrMany(S.And(NameSegment.Or(IndexSegment))).Then<IStatement>(ConvertSegments).Name(nameof(SingularQuerySegments));
public static readonly Parser<IStatement> RelSingularQuery = CurrentNodeIdentifier.And(SingularQuerySegments).Then<IStatement>(static x => new CurrentNode() { Child = x.Item2 }).Name(nameof(RelSingularQuery));
public static readonly Parser<IStatement> AbsSingularQuery = RootIdentifier.And(SingularQuerySegments).Then<IStatement>(static x => new RootNode() { Child = x.Item2 }).Name(nameof(AbsSingularQuery));
public static readonly Parser<IStatement> SingularQuery = RelSingularQuery.Or(AbsSingularQuery).Name(nameof(SingularQuery));
public static readonly Parser<IStatement> Comparable = Literal.Or(SingularQuery).Or(FunctionExpr).Name(nameof(Comparable));
public static readonly Parser<IStatement> ComparisonExpr = Comparable.And(S).And(ComparisonOp).And(S).And(Comparable).Then<IStatement>(static x => new OperatorStatement() { Left = x.Item1, Operator = x.Item3, Right = x.Item5 }).Name(nameof(ComparisonExpr));
public static readonly Parser<IStatement> FilterQuery = RelQuery.Or(JsonPathQuery).Name(nameof(FilterQuery));
public static readonly Parser<IStatement> FunctionArgument = FilterQuery.Or(LogicalExpr).Or(FunctionExpr).Or(Literal).Name(nameof(FunctionArgument));
public static readonly Parser<IStatement> TestExpr = Optional(LogicalNotOp.And(S)).And(FilterQuery.Or(FunctionExpr)).Then<IStatement>(static x => x.Item1.Item1 == '!' ? new UnaryOperaterStatement() { Operator = "!", Statement = x.Item2 } : x.Item2).Name(nameof(TestExpr));
public static readonly Parser<IStatement> BasicExpr = ParenExpr.Or(ComparisonExpr).Or(TestExpr).Name(nameof(BasicExpr));
public static readonly Parser<IStatement> LogicalAndExpr = BasicExpr.And(ZeroOrMany(S.And(Text("&&")).And(S).And(BasicExpr))).Then<IStatement>(static x =>
{
IStatement current = x.Item1;
if (x.Item2 != null && x.Item2.Count > 0)
{
foreach (var item in x.Item2)
{
current = new AndStatement() { Left = current, Right = item.Item4 };
}
}
return current;
}).Name(nameof(LogicalAndExpr));
public static readonly Parser<IStatement> LogicalOrExpr = LogicalAndExpr.And(ZeroOrMany(S.And(Text("||")).And(S).And(LogicalAndExpr))).Then<IStatement>(static x =>
{
IStatement current = x.Item1;
if (x.Item2 != null && x.Item2.Count > 0)
{
foreach (var item in x.Item2)
{
current = new OrStatement() { Left = current, Right = item.Item4 };
}
}
return current;
}).Name(nameof(LogicalOrExpr));
public static readonly Parser<IStatement> BracketedSelection = Char('[').And(S).And(Selector).And(ZeroOrMany(S.And(Char(',')).And(S).And(Selector))).And(S).And(Char(']'))
.Then<IStatement>(static x =>
{
var list = new List<IStatement> { x.Item3 };
if (x.Item4 != null)
list.AddRange(x.Item4.Select(y => y.Item4));
if (list.Count == 0)
return null;
return list.Count == 1 ? list[0] : new UnionSelectionStatement(list);
}).Name(nameof(BracketedSelection));
public static readonly Parser<IStatement> ChildSegment = BracketedSelection.Or(Char('.').And(WildcardSelector.Or(MemberNameShorthand)).Then<IStatement>(static x => x.Item2)).Name(nameof(ChildSegment));
public static readonly Parser<IStatement> DescendantSegment = Char('.').And(Char('.')).And(BracketedSelection.Or(WildcardSelector).Or(MemberNameShorthand)).Then<IStatement>(static x => new WildcardSelectorStatment() { Child = x.Item3 }).Name(nameof(DescendantSegment));
public static readonly Parser<IStatement> Segment = ChildSegment.Or(DescendantSegment).Name(nameof(Segment));
public static readonly Parser<IStatement> Parser;
static JsonPathParser()
{
LogicalExpr.Parser = LogicalOrExpr;
Segments.Parser = ZeroOrMany(S.And(Segment));
//MemberNameShorthand.Parser = NameFirst.And(ZeroOrMany(NameChar)).Then<IStatement>(static x => new Member { Name = x.Item1 + new string(x.Item2.ToArray()) });
FunctionExpr.Parser = FunctionName.And(Char('(')).And(S).And(Optional(FunctionArgument.And(ZeroOrMany(S.And(Char(',')).And(S).And(FunctionArgument))))).And(S).And(Char(')')).Then<IStatement>(static x =>
{
var args = new List<IStatement>();
if (x.Item4.Item1 != null)
{
args.Add(x.Item4.Item1);
}
if (x.Item4.Item2 != null)
{
args.AddRange(x.Item4.Item2.Select(y => y.Item4));
}
var func = new FunctionStatement()
{
Name = x.Item1,
Arguments = args.Count == 0 ? Array.Empty<IStatement>() : args.ToArray()
};
return func;
});
JsonPathQuery.Parser = RootIdentifier.And(Segments).Then<IStatement>(static x => new RootNode() { Child = ConvertSegments(x.Item2) });
Parser = JsonPathQuery.Eof().Name(nameof(Parser));
}
private static IStatement ConvertSegments(IReadOnlyList<(Nothing, IStatement)> x)
{
if (x == null || x.Count == 0)
{
return null;
}
else if (x.Count == 1)
return x[0].Item2;
else
{
var current = x.Last().Item2;
for (int i = x.Count - 2; i >= 0; i--)
{
if (x[i].Item2 is IParentStatement p)
{
var pp = p;
while (pp.Child != null)
{
var pc = p.Child as IParentStatement;
if (pc is null)
throw new NotSupportedException($"Cannot set child for statement of type {p.GetType().FullName}");
pp = pc;
}
pp.Child = current;
current = p;
}
else
{
throw new NotSupportedException($"Cannot set child for statement of type {x[i].Item2.GetType().FullName}");
}
}
return current;
}
}
}
性能测试
测试代码
[MemoryDiagnoser, GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
public class JsonPathBenchmarks
{
private object data = new
{
Num = -3.4,
Nu = null as string,
Array = new object[]
{
new { Name = "Alice", Age = 30 },
new { Name = "Bob", Age = 25 },
new { Name = "Charlie", Age = 35 }
},
};
private string path = "$.Array[1]['Name','Age']";
private string json;
private IStatement cache;
private readonly JsonPath pc;
public JsonPathBenchmarks()
{
json = JsonSerializer.Serialize(data);
JsonPathParser.Parser.TryParseResult(path, out var result, out var error);
cache = result.Value;
pc = JsonPath.Parse(path);
}
[Benchmark]
public object CacheTest()
{
return cache.EvaluateJson(json);
}
[Benchmark]
public object NoCacheTest()
{
JsonPathParser.Parser.TryParseResult(path, out var result, out var error);
return result.Value.EvaluateJson(json);
}
private Newtonsoft.Json.Linq.JToken testTo = Newtonsoft.Json.Linq.JToken.Parse("null");
[Benchmark]
public object NewtonsoftOnlyParseTest()
{
return testTo.SelectTokens(path);
}
[Benchmark]
public object NewtonsoftTest()
{
Newtonsoft.Json.Linq.JToken token = Newtonsoft.Json.Linq.JToken.Parse(json);
return token.SelectTokens(path);
}
[Benchmark]
public object JsonPathNetTest()
{
var p = JsonPath.Parse(path);
var instance = JsonNode.Parse(json);
return p.Evaluate(instance);
}
[Benchmark]
public object JsonPathNetCacheTest()
{
var instance = JsonNode.Parse(json);
return pc.Evaluate(instance);
}
[Benchmark]
public object JsonPathNetOnlyParseTest()
{
return JsonPath.Parse(path);
}
[Benchmark]
public object OnlyParseTest()
{
JsonPathParser.Parser.TryParseResult(path, out var result, out var error);
return result.Value;
}
}
效果
BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley)
Intel Core i7-10700 CPU 2.90GHz, 1 CPU, 16 logical and 8 physical cores
.NET SDK 9.0.304
[Host] : .NET 9.0.8 (9.0.8, 9.0.825.36511), X64 RyuJIT x86-64-v3
DefaultJob : .NET 9.0.8 (9.0.8, 9.0.825.36511), X64 RyuJIT x86-64-v3
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Allocated |
|---|---|---|---|---|---|---|
| CacheTest | 1,488.1 ns | 15.42 ns | 14.42 ns | 0.2537 | - | 2136 B |
| NoCacheTest | 2,650.5 ns | 11.44 ns | 10.14 ns | 0.4196 | - | 3528 B |
| NewtonsoftOnlyParseTest | 204.0 ns | 2.65 ns | 2.35 ns | 0.1128 | - | 944 B |
| NewtonsoftTest | 3,039.1 ns | 33.40 ns | 26.07 ns | 0.9651 | 0.0191 | 8088 B |
| JsonPathNetTest | 1,266.9 ns | 25.03 ns | 35.90 ns | 0.2728 | - | 2288 B |
| JsonPathNetCacheTest | 855.0 ns | 9.16 ns | 7.65 ns | 0.1469 | - | 1232 B |
| JsonPathNetOnlyParseTest | 346.3 ns | 3.19 ns | 2.66 ns | 0.1259 | - | 1056 B |
| OnlyParseTest | 1,057.0 ns | 5.24 ns | 4.90 ns | 0.1659 | - | 1392 B |
嗯，在 system.text.json 上解析 比 Newtonsoft.json 还是快了点，具体实现都是最基本的，没有像 jsonpath 那些做优化设计，性能感觉还是可以的
所有 实现放在 https://github.com/fs7744/Lmzzz
后续看什么时候有空搞搞 stream ，能行以后就这样偷懒搞吧
浙公网安备 33010602011771号