Fix several parsing issues.

This commit is contained in:
Joshua Bemenderfer
2021-12-30 17:45:53 -05:00
parent 38976cf29d
commit 7ba3f47785
164 changed files with 1240839 additions and 167 deletions

View File

@@ -2,12 +2,16 @@ import fs from 'fs/promises'
import fg from 'fast-glob'
import StreamZip from 'node-stream-zip'
import TestingTrend from './parser/testing-trend.js'
import path from 'path'
async function main() {
const sources = await fg(['./data/raw/*.zip'])
sources.sort()
const zips = sources.map(source => new StreamZip.async({ file: source }))
const zips = sources.map(source => ({
date: path.basename(source, path.extname(source)),
zip: new StreamZip.async({ file: source })
}))
await TestingTrend(zips)
}

View File

@@ -1,16 +1,17 @@
import mkdirp from 'mkdirp'
import path from 'path'
import fs from 'fs/promises'
import fs from 'fs'
import fsp from 'fs/promises'
import Papa from 'papaparse'
async function processSingleZip(zip) {
async function processSingleZip({ date, zip }) {
const output = {
directory: `./public/data/testing-trend`,
file: county => `${county}.json`
}
try {
await fs.rm(output.directory, { recursive: true })
await fsp.rm(output.directory, { recursive: true })
} catch (e) {}
const csv = await zip.entryData('pcr_antigen.csv').then(res => res.toString())
@@ -45,13 +46,14 @@ async function processSingleZip(zip) {
}
await mkdirp(output.directory)
await fs.writeFile(path.join(output.directory, output.file(county)), JSON.stringify(data))
await fsp.writeFile(path.join(output.directory, output.file(county)), JSON.stringify(data, null, '\t'))
}
}
function process (zips) {
return Promise.all(zips.map(zip => processSingleZip(zip)))
return processSingleZip(zips.at(-1))
}
export default process

BIN
data/raw/2021-12-30.zip Normal file

Binary file not shown.