Fix several parsing issues.

This commit is contained in:
Joshua Bemenderfer 2021-12-30 17:45:53 -05:00
parent 38976cf29d
commit 7ba3f47785
164 changed files with 1240839 additions and 167 deletions

View File

@ -2,12 +2,16 @@ import fs from 'fs/promises'
import fg from 'fast-glob' import fg from 'fast-glob'
import StreamZip from 'node-stream-zip' import StreamZip from 'node-stream-zip'
import TestingTrend from './parser/testing-trend.js' import TestingTrend from './parser/testing-trend.js'
import path from 'path'
async function main() { async function main() {
const sources = await fg(['./data/raw/*.zip']) const sources = await fg(['./data/raw/*.zip'])
sources.sort() sources.sort()
const zips = sources.map(source => new StreamZip.async({ file: source })) const zips = sources.map(source => ({
date: path.basename(source, path.extname(source)),
zip: new StreamZip.async({ file: source })
}))
await TestingTrend(zips) await TestingTrend(zips)
} }

View File

@ -1,16 +1,17 @@
import mkdirp from 'mkdirp' import mkdirp from 'mkdirp'
import path from 'path' import path from 'path'
import fs from 'fs/promises' import fs from 'fs'
import fsp from 'fs/promises'
import Papa from 'papaparse' import Papa from 'papaparse'
async function processSingleZip(zip) { async function processSingleZip({ date, zip }) {
const output = { const output = {
directory: `./public/data/testing-trend`, directory: `./public/data/testing-trend`,
file: county => `${county}.json` file: county => `${county}.json`
} }
try { try {
await fs.rm(output.directory, { recursive: true }) await fsp.rm(output.directory, { recursive: true })
} catch (e) {} } catch (e) {}
const csv = await zip.entryData('pcr_antigen.csv').then(res => res.toString()) const csv = await zip.entryData('pcr_antigen.csv').then(res => res.toString())
@ -45,13 +46,14 @@ async function processSingleZip(zip) {
} }
await mkdirp(output.directory) await mkdirp(output.directory)
await fs.writeFile(path.join(output.directory, output.file(county)), JSON.stringify(data))
await fsp.writeFile(path.join(output.directory, output.file(county)), JSON.stringify(data, null, '\t'))
} }
} }
function process (zips) { function process (zips) {
return Promise.all(zips.map(zip => processSingleZip(zip))) return processSingleZip(zips.at(-1))
} }
export default process export default process

BIN
data/raw/2021-12-30.zip Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More