Add initial support for toArray and array-based object values.

docs/experiments/parsers/v4/example.js

@@ -33,18 +33,22 @@ const lines = [
   ` vite ^3.2.3`,
   ` vitest ^0.24.5`,
   ``,
-  `author`,
+  `authors`,
-  ` name Joshua Bemenderfer`,
+  ` author`,
-  ` email josh@thederf.com`,
+  `  name Joshua Bemenderfer`,
-  ` `,
+  `  email josh@thederf.com`,
-  ` Further comments below. As I will now demonstrate, there is no simple`,
+  `  `,
-  `  even if embedded`,
+  `  Further comments below. As I will now demonstrate, there is no simple`,
-  ` way of dealing with this problem.`,
+  `   even if embedded`,
-  `author`,
+  `  way of dealing with this problem.`,
-  ` name Second Person`,
+  ` author`,
-  ` email second@secondperson.com`,
+  `  name Second Person`,
-  ` More text,`,
+  `  email second@secondperson.com`,
-  ` across two lines.`,
+  `  More text,`,
+  `  across two lines.`,
+  `list`,
+  ` - item1`,
+  ` - item2`
 ]
 
 // Schema
@@ -69,7 +73,7 @@ async function main() {
   const { toLineArray } = useDocument(createStringReader(lines))
   console.dir(await toLineArray(), { depth: null })
 
-  const { head, tail, each, match, toObject } = useDocument(createStringReader(lines))
+  const { head, tail, each, match, toArray, toObject } = useDocument(createStringReader(lines))
 
   const structure = await toObject({
     'name': true,
@@ -80,10 +84,9 @@ async function main() {
     }),
     'scripts': () => toObject({ '#any': true }),
     'devDependencies': () => toObject(),
-    'author': {
+    'authors': () => toArray({
-      type: 'collection',
+      'author': () => toObject({ name: true, email: true, '#text': true })
-      handle: () => toObject({ name: true, email: true, '#text': true })
+    }),
-    }
   })
 
   console.dir(structure, { depth: null })

packages/js/core/dist/document.cjs

@@ -1 +1 @@
-"use strict";Object.defineProperties(exports,{__esModule:{value:!0},[Symbol.toStringTag]:{value:"Module"}});const h=require("./parser.cjs");function O(m,b=" "){let o=0;const i=h.createLineData("",b);async function l(){switch(o){case 0:o=1;break;case 2:return o=1,!1}const e=await m.next();return e===null?(o=3,!0):(i.line=e,h.parseLine(i),!1)}const p=()=>o=2,a=()=>i.level,s=()=>i.line.slice(i.offsetHead),f=()=>i.line.slice(i.offsetHead,i.offsetTail),u=()=>i.line.slice(i.offsetTail),j=e=>e===f();async function y(e){const r=o===0?-1:a();for(;;){if(await l())return;if(a()<=r)return p();if(await e())return}}async function d(e=-1,r=[s()]){var t;return e===-1&&(e=a()+1),await l()?r:a()<e?(p(),r):(r.push(((t=i.line)==null?void 0:t.slice(e))||""),d(e,r))}async function v(e={}){const r={};let t={};return Object.keys(e).length?Object.keys(e).forEach(n=>{e[n]===!0&&(t[n]={type:"normal",handle:()=>u().trim()}),typeof e[n]=="function"&&(t[n]={type:"normal",handle:e[n]}),typeof e[n]=="object"&&(t[n]=e[n])}):t={"#any":{type:"normal",handle:()=>u().trim()}},await y(async()=>{const n=f();if(!n)return;const c=t[n]||t["#any"];if(!!c&&(c.type==="normal"?r[n]=await c.handle():c.type==="collection"?(r[n]||(r[n]=[]),r[n].push(await c.handle())):t!=null&&t["#text"]&&(r["#text"]=await d(a())),t&&Object.keys(t).every(w=>["collection"].includes(t[w].type)?!1:r[w]!==void 0)))return!0}),r}async function x(){const e=[["root",[]]];for(;!await l();){const r=a(),t=r+1,n=e[r];if(!n)continue;e.length=t;const c=e[t]=[s(),[]];n[1].push(c)}return e[0]}return{next:l,line:s,head:f,tail:u,level:a,match:j,each:y,blockAsText:d,toObject:v,toLineArray:x}}exports.useDocument=O;
+"use strict";Object.defineProperties(exports,{__esModule:{value:!0},[Symbol.toStringTag]:{value:"Module"}});const m=require("./parser.cjs");function g(w,b=" "){let c=0;const i=m.createLineData("",b);async function f(){switch(c){case 0:c=1;break;case 2:return c=1,!1}const e=await w.next();return e===null?(c=3,!0):(i.line=e,m.parseLine(i),!1)}const p=()=>c=2,o=()=>i.level,d=()=>i.line.slice(i.offsetHead),u=()=>i.line.slice(i.offsetHead,i.offsetTail),s=()=>i.line.slice(i.offsetTail),j=e=>e===u();async function y(e){const n=c===0?-1:o();for(;;){if(await f())return;if(o()<=n)return p();if(await e())return}}async function h(e=-1,n=[d()]){var r;return e===-1&&(e=o()+1),await f()?n:o()<e?(p(),n):(n.push(((r=i.line)==null?void 0:r.slice(e))||""),h(e,n))}async function x(e={},n=!1){const r=[];let t={};return Object.keys(e).length?Object.keys(e).forEach(a=>{e[a]===!0&&(t[a]={type:n?"collection":"normal",handle:()=>s().trim()}),typeof e[a]=="function"&&(t[a]={type:n?"collection":"normal",handle:e[a]}),typeof e[a]=="object"&&(t[a]=e[a])}):t={"#any":{type:n?"collection":"normal",handle:()=>s().trim()}},await y(async()=>{const a=u();if(!a)return;const l=t[a]||t["#any"];!l||(l.type==="normal"?r.push(await l.handle()):l.type==="collection"&&r.push({[a]:await l.handle()}))}),r}async function O(e={}){const n={};let r={};return Object.keys(e).length?Object.keys(e).forEach(t=>{t==="#text"?r[t]={type:"text",handle:()=>{}}:e[t]===!0?r[t]={type:"normal",handle:()=>s().trim()}:typeof e[t]=="function"?r[t]={type:"normal",handle:e[t]}:typeof e[t]=="object"&&(r[t]=e[t])}):r={"#any":{type:"normal",handle:()=>s().trim()}},await y(async()=>{const t=u();if(!t)return;const a=r[t]||r["#any"]||r["#text"];if(!!a&&(a.type==="normal"?n[t]=await a.handle():a.type==="collection"?(n[t]||(n[t]=[]),n[t].push(await a.handle())):a.type==="text"&&(n["#text"]=await h(o())),r&&Object.keys(r).every(l=>["collection"].includes(r[l].type)?!1:n[l]!==void 0)))return!0}),n}async function v(){const e=[["root",[]]];for(;!await f();){const n=o(),r=n+1,t=e[n];if(!t)continue;e.length=r;const a=e[r]=[d(),[]];t[1].push(a)}return e[0]}return{next:f,line:d,head:u,tail:s,level:o,match:j,each:y,blockAsText:h,toObject:O,toArray:x,toLineArray:v}}exports.useDocument=g;

packages/js/core/dist/document.js

@@ -1,8 +1,8 @@
-import { parseLine as v, createLineData as O } from "./parser.js";
+import { parseLine as v, createLineData as H } from "./parser.js";
-function D(h, m = " ") {
+function g(p, w = " ") {
   let c = 0;
-  const r = O("", m);
+  const i = H("", w);
-  async function l() {
+  async function s() {
     switch (c) {
       case 0:
         c = 1;
@@ -10,64 +10,79 @@ function D(h, m = " ") {
       case 2:
         return c = 1, !1;
     }
-    const e = await h.next();
+    const e = await p.next();
-    return e === null ? (c = 3, !0) : (r.line = e, v(r), !1);
+    return e === null ? (c = 3, !0) : (i.line = e, v(i), !1);
   }
-  const y = () => c = 2, a = () => r.level, f = () => r.line.slice(r.offsetHead), s = () => r.line.slice(r.offsetHead, r.offsetTail), u = () => r.line.slice(r.offsetTail), b = (e) => e === s();
+  const m = () => c = 2, l = () => i.level, y = () => i.line.slice(i.offsetHead), u = () => i.line.slice(i.offsetHead, i.offsetTail), f = () => i.line.slice(i.offsetTail), b = (e) => e === u();
-  async function w(e) {
+  async function h(e) {
-    const i = c === 0 ? -1 : a();
+    const n = c === 0 ? -1 : l();
     for (; ; ) {
-      if (await l())
+      if (await s())
         return;
-      if (a() <= i)
+      if (l() <= n)
-        return y();
+        return m();
       if (await e())
         return;
     }
   }
-  async function p(e = -1, i = [f()]) {
+  async function d(e = -1, n = [y()]) {
-    var t;
+    var a;
-    return e === -1 && (e = a() + 1), await l() ? i : a() < e ? (y(), i) : (i.push(((t = r.line) == null ? void 0 : t.slice(e)) || ""), p(e, i));
+    return e === -1 && (e = l() + 1), await s() ? n : l() < e ? (m(), n) : (n.push(((a = i.line) == null ? void 0 : a.slice(e)) || ""), d(e, n));
   }
-  async function j(e = {}) {
+  async function j(e = {}, n = !1) {
-    const i = {};
+    const a = [];
     let t = {};
-    return Object.keys(e).length ? Object.keys(e).forEach((n) => {
+    return Object.keys(e).length ? Object.keys(e).forEach((r) => {
-      e[n] === !0 && (t[n] = { type: "normal", handle: () => u().trim() }), typeof e[n] == "function" && (t[n] = { type: "normal", handle: e[n] }), typeof e[n] == "object" && (t[n] = e[n]);
+      e[r] === !0 && (t[r] = { type: n ? "collection" : "normal", handle: () => f().trim() }), typeof e[r] == "function" && (t[r] = { type: n ? "collection" : "normal", handle: e[r] }), typeof e[r] == "object" && (t[r] = e[r]);
-    }) : t = { "#any": { type: "normal", handle: () => u().trim() } }, await w(async () => {
+    }) : t = { "#any": { type: n ? "collection" : "normal", handle: () => f().trim() } }, await h(async () => {
-      const n = s();
+      const r = u();
-      if (!n)
+      if (!r)
         return;
-      const o = t[n] || t["#any"];
+      const o = t[r] || t["#any"];
-      if (!!o && (o.type === "normal" ? i[n] = await o.handle() : o.type === "collection" ? (i[n] || (i[n] = []), i[n].push(await o.handle())) : t != null && t["#text"] && (i["#text"] = await p(a())), t && Object.keys(t).every((d) => ["collection"].includes(t[d].type) ? !1 : i[d] !== void 0)))
+      !o || (o.type === "normal" ? a.push(await o.handle()) : o.type === "collection" && a.push({ [r]: await o.handle() }));
-        return !0;
+    }), a;
-    }), i;
   }
-  async function x() {
+  async function x(e = {}) {
+    const n = {};
+    let a = {};
+    return Object.keys(e).length ? Object.keys(e).forEach((t) => {
+      t === "#text" ? a[t] = { type: "text", handle: () => {
+      } } : e[t] === !0 ? a[t] = { type: "normal", handle: () => f().trim() } : typeof e[t] == "function" ? a[t] = { type: "normal", handle: e[t] } : typeof e[t] == "object" && (a[t] = e[t]);
+    }) : a = { "#any": { type: "normal", handle: () => f().trim() } }, await h(async () => {
+      const t = u();
+      if (!t)
+        return;
+      const r = a[t] || a["#any"] || a["#text"];
+      if (!!r && (r.type === "normal" ? n[t] = await r.handle() : r.type === "collection" ? (n[t] || (n[t] = []), n[t].push(await r.handle())) : r.type === "text" && (n["#text"] = await d(l())), a && Object.keys(a).every((o) => ["collection"].includes(a[o].type) ? !1 : n[o] !== void 0)))
+        return !0;
+    }), n;
+  }
+  async function O() {
     const e = [["root", []]];
-    for (; !await l(); ) {
+    for (; !await s(); ) {
-      const i = a(), t = i + 1, n = e[i];
+      const n = l(), a = n + 1, t = e[n];
-      if (!n)
+      if (!t)
         continue;
-      e.length = t;
+      e.length = a;
-      const o = e[t] = [f(), []];
+      const r = e[a] = [y(), []];
-      n[1].push(o);
+      t[1].push(r);
     }
     return e[0];
   }
   return {
-    next: l,
+    next: s,
-    line: f,
+    line: y,
-    head: s,
+    head: u,
-    tail: u,
+    tail: f,
-    level: a,
+    level: l,
     match: b,
-    each: w,
+    each: h,
-    blockAsText: p,
+    blockAsText: d,
-    toObject: j,
+    toObject: x,
-    toLineArray: x
+    toArray: j,
+    toLineArray: O
   };
 }
 export {
-  D as useDocument
+  g as useDocument
 };

packages/js/core/src/document.ts

@@ -17,6 +17,7 @@ type Document = {
   match: (matchHead: string) => boolean,
   each: (handler: Function) => void,
   blockAsText: (startLevel: number, lines?: string[]) => Promise<Array<string>>,
+  toArray: (inputMatchers: { [key: string]: Function|boolean|{ type: string, handle: Function } }, collection: boolean) => Promise<[{ [key: string]: any }?]>,
   toObject: (matchers?: { [key: string]: Function|boolean }) => { [key: string]: any },
   toLineArray(): Promise<LineArray>
 }
@@ -84,7 +85,44 @@ export function useDocument (reader: Reader, indent: string = ' '): Document {
     return blockAsText(startLevel, blockLines)
   }
 
-  async function toObject (inputMatchers: { [key: string]: Function|boolean|{ type: string, handle: Function } } = {}) {
+  // Currently a modified copy of toObject. Has lots of room for simplification.
+  async function toArray (inputMatchers: { [key: string]: Function|boolean|{ type: string, handle: Function } } = {}, collection: boolean = false): Promise<[{ [key: string]: any }?]> {
+    const arr: [{ [key: string]: any }?] = []
+
+    let matchers: { [key: string]: {type: string, handle: Function } } = {}
+
+    // Normalize the matchers to an object-based format despite allowing flexible input types for convenience.
+    // TODO: Decide whether to enforce verbose input once a DSL has been created.
+    if (!Object.keys(inputMatchers).length) {
+      // Default matcher
+      matchers = { '#any': { type: collection ? 'collection' : 'normal', handle: () => tail().trim() } }
+    } else {
+      Object.keys(inputMatchers).forEach(key => {
+        // If a matcher is specified as `true`, treat as a key-value pair where { [head]: tail }
+        if(inputMatchers[key] === true) matchers[key] = { type: collection ? 'collection' : 'normal', handle: () => tail().trim() }
+        // If a matcher is specified as a function, treat as a key-value pair where { [head]: handle() }
+        if (typeof inputMatchers[key] === 'function') matchers[key] = { type: collection ? 'collection' : 'normal', handle: inputMatchers[key] as Function }
+        // If a matcher is specified as an object, allow customization of the type and handle for various cases.
+        if (typeof inputMatchers[key] === 'object') matchers[key] = inputMatchers[key] as { type: string, handle: Function }
+      })
+    }
+
+    await each(async () => {
+      const currHead = head()
+      if (!currHead) return
+
+      const currMatcher = matchers[currHead] || matchers['#any']
+      if (!currMatcher) return
+
+      // Normal - Outputs values directly into the array, removing their keys.
+      if (currMatcher.type === 'normal') arr.push(await currMatcher.handle())
+      // Collection - Outputs values as { head: value } objects into the array, preserving their keys.
+      else if (currMatcher.type === 'collection') arr.push({ [currHead]: await currMatcher.handle() })
+    })
+    return arr
+  }
+
+  async function toObject (inputMatchers: { [key: string]: Function|boolean|{ type: string, handle: Function } } = {}): Promise<{ [key: string]: any }> {
     const obj: { [key: string]: any } = {}
 
     let matchers: { [key: string]: {type: string, handle: Function } } = {}
@@ -96,12 +134,13 @@ export function useDocument (reader: Reader, indent: string = ' '): Document {
       matchers = { '#any': { type: 'normal', handle: () => tail().trim() } }
     } else {
       Object.keys(inputMatchers).forEach(key => {
+        if (key === '#text') matchers[key] = { type: 'text', handle: () => {} }
         // If a matcher is specified as `true`, treat as a key-value pair where { [head]: tail }
-        if(inputMatchers[key] === true) matchers[key] = { type: 'normal', handle: () => tail().trim() }
+        else if (inputMatchers[key] === true) matchers[key] = { type: 'normal', handle: () => tail().trim() }
         // If a matcher is specified as a function, treat as a key-value pair where { [head]: handle() }
-        if (typeof inputMatchers[key] === 'function') matchers[key] = { type: 'normal', handle: inputMatchers[key] as Function }
+        else if (typeof inputMatchers[key] === 'function') matchers[key] = { type: 'normal', handle: inputMatchers[key] as Function }
         // If a matcher is specified as an object, allow customization of the type and handle for various cases.
-        if (typeof inputMatchers[key] === 'object') matchers[key] = inputMatchers[key] as { type: string, handle: Function }
+        else if (typeof inputMatchers[key] === 'object') matchers[key] = inputMatchers[key] as { type: string, handle: Function }
       })
     }
 
@@ -109,19 +148,19 @@ export function useDocument (reader: Reader, indent: string = ' '): Document {
       const currHead = head()
       if (!currHead) return
 
-      const currentMatcher = matchers[currHead] || matchers['#any']
+      const currMatcher = matchers[currHead] || matchers['#any'] || matchers['#text']
-      if (!currentMatcher) return
+      if (!currMatcher) return
 
-      if (currentMatcher.type === 'normal') obj[currHead] = await currentMatcher.handle()
+      if (currMatcher.type === 'normal') obj[currHead] = await currMatcher.handle()
       // Allows matching the same key more than once.
-      else if (currentMatcher.type === 'collection') {
+      else if (currMatcher.type === 'collection') {
         if (!obj[currHead]) obj[currHead] = []
-        obj[currHead].push(await currentMatcher.handle())
+        obj[currHead].push(await currMatcher.handle())
       }
       // If matchers[currHead] or matchers[#any] is a function, set object key to its output.
       // If we get to this point and matchers[#text] is set, parse all remaining block contents as text.
       // TODO: I still don't like this.
-      else if (matchers?.['#text']) obj['#text'] = await blockAsText(level())
+      else if (currMatcher.type === 'text') obj['#text'] = await blockAsText(level())
 
       // Bail early as soon as we know all keys have been matched.
       if (matchers && Object.keys(matchers).every(key => {
@@ -167,6 +206,7 @@ export function useDocument (reader: Reader, indent: string = ' '): Document {
     each,
     blockAsText,
     toObject,
+    toArray,
     toLineArray,
   }
 }