From a11e6780fbb8bd4143dfec44e2ce147b795772d8 Mon Sep 17 00:00:00 2001 From: Kai Stevenson Date: Mon, 3 Nov 2025 21:24:05 -0800 Subject: fix lexer depth limit --- src/lang/core/eval.ts | 2 +- src/lang/core/lexer.ts | 61 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 17 deletions(-) (limited to 'src/lang') diff --git a/src/lang/core/eval.ts b/src/lang/core/eval.ts index 981aef8..9181c81 100644 --- a/src/lang/core/eval.ts +++ b/src/lang/core/eval.ts @@ -105,7 +105,7 @@ export type GetEvaluatedChildren< : never; const input = - `map(arr("hello","world"),fn(s,i,add(tostring(i),":",s)))` as const; + `map(arr("hello", "world"), fn(s, i, add(tostring(i), ":", s)))` as const; const lex_result = null as unknown as Lex; const parse_result = null as unknown as Parse; const eval_result = null as unknown as Evaluate; diff --git a/src/lang/core/lexer.ts b/src/lang/core/lexer.ts index 33a408a..567964f 100644 --- a/src/lang/core/lexer.ts +++ b/src/lang/core/lexer.ts @@ -40,23 +40,52 @@ export type ProcessNameCollection< export type IsOpen = T extends `${TokenType.OPEN_PAREN}` ? true : false; export type IsClose = T extends `${TokenType.CLOSE_PAREN}` ? true : false; -export type _Lex = - Ctx["next"] extends `${infer Head}${infer Tail}` - ? IsWhitespace extends true - ? _Lex> - : IsOpen extends true - ? _Lex>> - : IsClose extends true - ? _Lex>> - : _Lex<{ +export type ChunkedLex< + Ctx extends LexerCtx, + Depth extends any[] = [] +> = Depth["length"] extends 50 + ? Ctx & { + endChunk: true; + } + : Ctx["next"] extends `${infer Head}${infer Tail}` + ? IsWhitespace extends true + ? ChunkedLex, [0, ...Depth]> + : IsOpen extends true + ? ChunkedLex< + ProcessNameCollection>, + [0, ...Depth] + > + : IsClose extends true + ? ChunkedLex< + ProcessNameCollection>, + [0, ...Depth] + > + : ChunkedLex< + { next: Tail; nameCollection: `${Ctx["nameCollection"]}${Head}`; tokens: Ctx["tokens"]; - }> - : Ctx["tokens"]; + }, + [0, ...Depth] + > + : Ctx; -export type Lex = _Lex<{ - next: `${Raw};`; - tokens: []; - nameCollection: ""; -}>; +export type InnerLex< + Next extends string, + NameCollection extends LexerCtx["nameCollection"] = "", + AccTokens extends Token[] = [] +> = Next extends "" + ? AccTokens + : ChunkedLex<{ + next: Next; + tokens: []; + nameCollection: NameCollection; + }> extends infer U + ? U extends LexerCtx & { endChunk: true } + ? InnerLex + : U extends LexerCtx + ? [...AccTokens, ...U["tokens"]] + : never + : never; + +export type Lex = InnerLex; -- cgit v1.2.3-70-g09d2