Skip to content

Commit 29fe8eb

Browse files
kbowermakbowerma
committed
inital commit of 30061038
0 parents  commit 29fe8eb

22 files changed

+3403
-0
lines changed

‎.eslintrc

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"extends": "airbnb-base",
3+
"rules": {
4+
"no-use-before-define": 0,
5+
"strict": 0,
6+
"comma-dangle": [2, "never"],
7+
"func-names": 0,
8+
"no-underscore-dangle": 0,
9+
"prefer-rest-params": 0,
10+
"no-param-reassign": 0,
11+
"prefer-template": 0,
12+
"new-cap": 0,
13+
"global-require": 0,
14+
"consistent-return": 0,
15+
"max-len": [2, 200],
16+
"camelcase": 0,
17+
"import/no-dynamic-require": 0
18+
},
19+
"parserOptions": {
20+
"ecmaVersion": 6
21+
},
22+
"globals": {
23+
"describe": true,
24+
"it": true,
25+
"before": true,
26+
"beforeEach": true,
27+
"after": true,
28+
"afterEach": true
29+
}
30+
}

‎.gitignore

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
node_modules
2+
*.zip
3+
# Logs
4+
logs
5+
*.log
6+
npm-debug.log*
7+
8+
#kyle
9+
.DS_Store
10+
*.csv
11+
12+
# Runtime data
13+
pids
14+
*.pid
15+
*.seed
16+
*.pid.lock
17+
18+
# Directory for instrumented libs generated by jscoverage/JSCover
19+
lib-cov
20+
21+
# Coverage directory used by tools like istanbul
22+
coverage
23+
24+
# nyc test coverage
25+
.nyc_output
26+
27+
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
28+
.grunt
29+
30+
# node-waf configuration
31+
.lock-wscript
32+
33+
# Compiled binary addons (http://nodejs.org/api/addons.html)
34+
build/Release
35+
36+
# Dependency directories
37+
node_modules
38+
jspm_packages
39+
40+
# Optional npm cache directory
41+
.npm
42+
43+
# Optional eslint cache
44+
.eslintcache
45+
46+
# Optional REPL history
47+
.node_repl_history

‎README.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# VA ONLINE MEMORIAL - DATA IMPORT & SYNC
2+
3+
## Dependencies
4+
- [Nodejs](https://nodejs.org/en/)
5+
- [PostgreSQL](https://www.postgresql.org/)
6+
- [eslint](http://eslint.org/)
7+
8+
## Configuration
9+
- Edit configuration in `config/default.json` and
10+
- custom environment variables names in `config/custom-environment-variables.json`,
11+
12+
## Application constants
13+
14+
- Application constants can be configured in `./constants.js`
15+
16+
## Available tools
17+
18+
- Since the data we need to download and process is huge it's better (/ safer) to use 2 different tools instead of one single script so in case that something goes wrong during processing, we'll minimise the damage.
19+
20+
### Download datasets
21+
22+
- Run `npm run download-data` to download all available datasets.
23+
- The datasets will be stored in the configured directory.
24+
- Old data will be replaced.
25+
- This operation does not affect the database.
26+
27+
### Import data from downloaded files
28+
29+
- Run `npm run import-data` to import all data using the downloaded files from the previous step.
30+
31+
## Local Deployment
32+
33+
*Before starting the application, make sure that PostgreSQL is running and you have configured everything correctly in `config/default.json`*
34+
35+
- Install dependencies `npm i`
36+
- Run lint check `npm run lint`
37+
- Start app `npm start`. This will run all tools in the following sequence:
38+
39+
`npm run download-data` => `npm run import-data`
40+
41+
*The application will print progress information and the results in the terminal.*
42+
43+
## Verification
44+
45+
- To verify that the data is imported, you can use the [pgAdmin](https://www.pgadmin.org/) tool and browser the database.
46+
47+
## Notes:
48+
49+
- The total size of all datasets is > 1.5GB so it will take quite some time, depending on your internet connection, to finish the operation.
50+
- `max_old_space_size` has been set to *4096MB* to allow parse/process such huge data files without any issues. The app will clean the memory right after using the data to prevent memory/heap leaks.
51+
- The dataset for `FOREIGN ADDRESSES` doesn't have a header in the CSV file and it has slightly different format (it has an extra column). The app handles all datasets without any issue.

‎common/logger.js

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
'use strict';
2+
3+
/*
4+
* Copyright (C) 2017 Topcoder Inc., All Rights Reserved.
5+
*/
6+
7+
/**
8+
* This module contains the winston logger configuration.
9+
*/
10+
11+
const winston = require('winston');
12+
const config = require('config');
13+
const chalk = require('chalk');
14+
15+
const logger = new (winston.Logger)({
16+
transports: [
17+
new (winston.transports.Console)({
18+
level: config.logLevel,
19+
timestamp: () => new Date().toISOString(),
20+
formatter(options) {
21+
const message = options.message || '';
22+
23+
let meta = '';
24+
if (options.meta && Object.keys(options.meta).length) {
25+
meta = '\n\t' + JSON.stringify(options.meta);
26+
}
27+
28+
let level = options.level.toUpperCase();
29+
switch (level) {
30+
case 'INFO':
31+
level = chalk.cyan(level);
32+
break;
33+
case 'WARN':
34+
level = chalk.yellow(level);
35+
break;
36+
case 'ERROR':
37+
level = chalk.red(level);
38+
break;
39+
default:
40+
break;
41+
}
42+
43+
return `[${options.timestamp()}][${level}] ${message} ${meta}`;
44+
}
45+
})
46+
]
47+
});
48+
49+
module.exports = logger;

‎config/custom-environment-variables.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"dataset_url": "DATASET_URL",
3+
"logLevel": "LOG_LEVEL",
4+
"dbConfig": {
5+
"db_url": "DATABASE_URL"
6+
},
7+
"downloadPath": "DOWNLOAD_PATH"
8+
}

‎config/default.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"dataset_url": "https://www.data.va.gov/data.json",
3+
"logLevel": "info",
4+
"dbConfig": {
5+
"db_url": "postgres://user:pass@localhost:5432/va"
6+
},
7+
"downloadPath": "downloads"
8+
}

‎constants.js

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
'use strict';
2+
3+
/*
4+
* Copyright (c) 2017 Topcoder, Inc. All rights reserved.
5+
*/
6+
7+
/**
8+
* Application constants
9+
*/
10+
11+
// The accepted program codes
12+
const acceptedProgramCodes = [
13+
'029:001'
14+
];
15+
16+
// The accepted keywords
17+
const acceptedKeywords = [
18+
'burial data'
19+
];
20+
21+
// The accepted file format
22+
const acceptedFormat = 'csv';
23+
24+
// Entry names that should be ignored
25+
const ignoredNames = [
26+
// Source of cemeteries data
27+
'VA Cemeteries - Address, Location, Contact Information, Burial Space'
28+
];
29+
30+
// CSV headers
31+
const csvHeaders = [
32+
'd_first_name',
33+
'd_mid_name',
34+
'd_last_name',
35+
'd_suffix',
36+
'd_birth_date',
37+
'd_death_date',
38+
'section_id',
39+
'row_num',
40+
'site_num',
41+
'cem_name',
42+
'cem_addr_one',
43+
'cem_addr_two',
44+
'city',
45+
'state',
46+
'zip',
47+
'cem_url',
48+
'cem_phone',
49+
'relationship',
50+
'v_first_name',
51+
'v_mid_name',
52+
'v_last_name',
53+
'v_suffix',
54+
'branch',
55+
'rank',
56+
'war'
57+
];
58+
59+
module.exports = {
60+
acceptedProgramCodes,
61+
acceptedKeywords,
62+
acceptedFormat,
63+
ignoredNames,
64+
csvHeaders
65+
};

‎csvCounts.txt

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
0 csvCounts.txt
2+
21872 ngl_alabama.csv
3+
8880 ngl_alaska.csv
4+
96485 ngl_arizona.csv
5+
64296 ngl_arkansas.csv
6+
741842 ngl_california.csv
7+
141187 ngl_colorado.csv
8+
1 ngl_connecticut.csv
9+
21309 ngl_delaware.csv
10+
305774 ngl_florida.csv
11+
4313 ngl_foreign_addresses.csv
12+
57960 ngl_georgia.csv
13+
73123 ngl_hawaii.csv
14+
7362 ngl_idaho.csv
15+
135405 ngl_illinois.csv
16+
24972 ngl_indiana.csv
17+
9565 ngl_iowa.csv
18+
77240 ngl_kansas.csv
19+
60459 ngl_kentucky.csv
20+
35590 ngl_louisiana.csv
21+
30708 ngl_maine.csv
22+
137464 ngl_maryland.csv
23+
84708 ngl_massachusetts.csv
24+
71418 ngl_michigan.csv
25+
234249 ngl_minnesota.csv
26+
41613 ngl_mississippi.csv
27+
252711 ngl_missouri.csv
28+
3784 ngl_montana.csv
29+
12859 ngl_nebraska.csv
30+
49952 ngl_nevada.csv
31+
10217 ngl_new_hampshire.csv
32+
113334 ngl_new_jersey.csv
33+
65241 ngl_new_mexico.csv
34+
683640 ngl_new_york.csv
35+
40412 ngl_north_carolina.csv
36+
7678 ngl_north_dakota.csv
37+
90253 ngl_ohio.csv
38+
31285 ngl_oklahoma.csv
39+
202501 ngl_oregon.csv
40+
94842 ngl_pennsylvania.csv
41+
28365 ngl_rhode_island.csv
42+
45634 ngl_south_carolina.csv
43+
30243 ngl_south_dakota.csv
44+
207440 ngl_tennessee.csv
45+
384140 ngl_texas.csv
46+
90157 ngl_usa_territoties.csv
47+
6017 ngl_utah.csv
48+
28 ngl_vermont.csv
49+
122320 ngl_virginia.csv
50+
56423 ngl_washington.csv
51+
465 ngl_washingtondc.csv
52+
9229 ngl_west_virginia.csv
53+
67260 ngl_wisconsin.csv
54+
1813 ngl_wyoming_0.csv
55+
5196038 total

‎models/Branch.js

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
'use strict';
2+
3+
/*
4+
* Copyright (c) 2017 Topcoder, Inc. All rights reserved.
5+
*/
6+
7+
/*
8+
* Branch model definition
9+
*/
10+
module.exports = (sequelize, DataTypes) => sequelize.define('Branch', {
11+
value: {
12+
type: DataTypes.STRING,
13+
allowNull: null,
14+
primaryKey: true,
15+
unique: true
16+
}
17+
}, {
18+
timestamps: false
19+
});

‎models/Burial.js

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
'use strict';
2+
3+
/*
4+
* Copyright (c) 2017 Topcoder, Inc. All rights reserved.
5+
*/
6+
7+
/*
8+
* Burial model definition
9+
*/
10+
module.exports = (sequelize, DataTypes) => sequelize.define('Burial', {
11+
d_id: { type: DataTypes.STRING, allowNull: false, primaryKey: true },
12+
cem_id: { type: DataTypes.STRING, allowNull: false },
13+
section_id: DataTypes.STRING,
14+
row_num: DataTypes.STRING,
15+
site_num: DataTypes.STRING
16+
}, {
17+
timestamps: false
18+
});

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy