You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

342 lines
38 KiB

3 years ago
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = y[op[0] & 2 ? "return" : op[0] ? "throw" : "next"]) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [0, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
var src_1 = __importDefault(require("../src"));
var assert_1 = __importDefault(require("assert"));
var fs = require("fs");
var sinon_1 = require("sinon");
var sb = sinon_1.sandbox.create();
describe("testCSVConverter3", function () {
afterEach(function () {
sb.restore();
});
it("should parse large csv file with UTF-8 without spliting characters", function (done) {
var testData = __dirname + "/data/large-utf8.csv";
var rs = fs.createReadStream(testData);
var csvConverter = src_1.default({});
var count = 0;
csvConverter.preRawData(function (csvRawData) {
assert_1.default(csvRawData.charCodeAt(0) < 2000);
return csvRawData;
});
csvConverter.on("data", function () {
count++;
});
csvConverter.then(function () {
assert_1.default(count === 5290);
done();
});
rs.pipe(csvConverter);
});
it("should setup customise type convert function", function (done) {
src_1.default({
checkType: true,
colParser: {
"column1": "string",
"column5": function (item, head, resultRow, row, i) {
assert_1.default.equal(item, '{"hello":"world"}');
assert_1.default.equal(head, "column5"),
assert_1.default(resultRow);
assert_1.default(row);
assert_1.default.equal(i, 5);
return "hello world";
}
}
})
.fromFile(__dirname + "/data/dataWithType")
.subscribe(function (json) {
assert_1.default.equal(typeof json.column1, "string");
assert_1.default.equal(json.column5, "hello world");
assert_1.default.strictEqual(json["name#!"], false);
assert_1.default.strictEqual(json["column9"], true);
})
.on('done', function () {
done();
});
});
it("should accept pipe as quote", function (done) {
src_1.default({
quote: "|",
output: "csv"
})
.fromFile(__dirname + "/data/pipeAsQuote")
.subscribe(function (csv) {
assert_1.default.equal(csv[2], "blahhh, blah");
})
.on('done', function () {
done();
});
});
it("emit file not exists error when try to open a non-exists file", function () {
var called = false;
var cb = sb.spy(function (err) {
assert_1.default(err.toString().indexOf("File does not exist") > -1);
});
return src_1.default()
.fromFile("somefile")
.subscribe(function (csv) {
})
.on("error", cb)
.then(function () {
assert_1.default(false);
}, function (err) {
assert_1.default.equal(cb.callCount, 1);
});
});
it("should include column that is both included and excluded", function () {
return src_1.default({
includeColumns: /b/,
ignoreColumns: /a|b/
})
.fromString("a,b,c\n1,2,3\n4,5,6")
.subscribe(function (d) {
assert_1.default(d.b);
assert_1.default(!d.a);
});
});
it("should allow async preLine hook", function () {
return src_1.default()
.preFileLine(function (line) {
return new Promise(function (resolve, reject) {
setTimeout(function () {
resolve(line + "changed");
}, 20);
});
})
.fromString("a,b\n1,2")
.subscribe(function (d) {
assert_1.default(d.bchanged);
assert_1.default.equal(d.bchanged, "2changed");
});
});
it("should allow async subscribe function", function () {
return src_1.default({ trim: true })
.fromString("a,b,c\n 1,2,3\n 4,5,6")
.subscribe(function (d) {
return new Promise(function (resolve, reject) {
setTimeout(function () {
d.a = 10;
resolve();
}, 20);
});
})
.then(function (d) {
assert_1.default.equal(d[0].a, 10);
assert_1.default.equal(d[1].a, 10);
});
});
it("should propagate value to next then", function () {
return src_1.default({ trim: true })
.fromString("a,b,c\n 1,2,3\n 4,5,6")
.then(undefined, undefined)
.then(function (d) {
assert_1.default.equal(d.length, 2);
assert_1.default.equal(d[0].a, "1");
});
});
it("should propagate error to next then", function () {
return src_1.default({ trim: true })
.fromFile(__dirname + "/data/dataWithUnclosedQuotes")
.then(undefined, undefined)
.then(function () {
assert_1.default(false);
}, function (err) {
assert_1.default(err);
assert_1.default.equal(err.err, "unclosed_quote");
});
});
it("should fallback to text is number can not be parsed", function () {
return src_1.default({
colParser: {
"a": "number"
}
})
.fromString("a,b,c\n 1,2,3\n fefe,5,6")
.then(function (d) {
assert_1.default.strictEqual(d[0].a, 1);
assert_1.default.equal(d[1].a, "fefe");
});
});
it("should omit a column", function () {
return src_1.default({
colParser: {
"a": "omit"
}
})
.fromString("a,b,c\n 1,2,3\n fefe,5,6")
.then(function (d) {
assert_1.default.strictEqual(d[0].a, undefined);
assert_1.default.equal(d[1].a, undefined);
});
});
it("could turn off quote and should trim even quote is turned off", function () {
return src_1.default({
quote: "off",
trim: true
})
.fromString("a,b,c\n \"1\",\"2\",\"3\"\n \"fefe,5\",6")
.then(function (d) {
assert_1.default.equal(d[0].a, '"1"');
assert_1.default.equal(d[0].b, '"2"');
assert_1.default.equal(d[1].a, '"fefe');
assert_1.default.equal(d[1].b, '5"');
});
});
it("should allow ignoreEmpty with checkColumn", function () {
return src_1.default({
checkColumn: true,
ignoreEmpty: true
})
.fromString("date,altitude,airtime\n 2016-07-08,2000,23\n \n 2016-07-09,3000,43")
.then(function (data) {
}, function (err) {
console.log(err);
assert_1.default(!err);
});
});
it("should allow quotes without content", function () {
var data = "a|^^|^b^";
return src_1.default({
delimiter: '|',
quote: '^',
noheader: true,
})
.fromString(data)
.then(function (jsonObj) {
assert_1.default.equal(jsonObj[0].field2, "");
});
});
it("should parse header with quotes correctly", function () {
var testData = __dirname + "/data/csvWithUnclosedHeader";
return src_1.default({
headers: ["exam_date", "sample_no", "status", "sample_type", "patient_id", "last_name", "first_name", "gender_of_patient", "patient_birth_date", "patient_note", "patient_department", "accession_number", "sample_site", "physician", "operator", "department", "note", "test_order_code", "draw_time", "approval_status", "approval_time", "report_layout", "patient_account_number", "none_1", "errors_detected_during_measurement", "age", "error_code_01", "weight", "error_code_02", "height", "error_code_03", "hcg_beta_p", "error_code_04", "troponin_i_p", "error_code_05", "ck_mb_p", "error_code_06", "d_dimer_p", "error_code_07", "hscrp_p", "error_code_08", "myoglobin_p", "error_code_09", "nt_probnp", "error_code_10", "crp", "error_code_11", "bnp", "error_code_12", "tnt", "error_code_13", "demo_p", "error_code_14", "pct", "error_code_15"]
})
.fromFile(testData)
.then(function (d) {
assert_1.default.equal(d.length, 2);
assert_1.default.equal(d[0].sample_no, "12669");
});
});
it("should stream json string correctly", function (done) {
var data = "a,b,c\n1,2,3\n4,5,6";
var hasLeftBracket = false;
var hasRightBracket = false;
src_1.default({
downstreamFormat: "array"
})
.fromString(data)
.on("data", function (d) {
var str = d.toString();
if (str[0] === "[" && str.length === 2) {
hasLeftBracket = true;
}
else if (str[0] === "]" && str.length === 2) {
hasRightBracket = true;
}
else {
assert_1.default.equal(str[str.length - 2], ",");
}
})
.on("end", function () {
assert_1.default.equal(hasLeftBracket, true);
assert_1.default.equal(hasRightBracket, true);
done();
});
});
it("should stream json line correctly", function (done) {
var data = "a,b,c\n1,2,3\n4,5,6";
src_1.default({
downstreamFormat: "line"
})
.fromString(data)
.on("data", function (d) {
var str = d.toString();
assert_1.default.notEqual(str[str.length - 2], ",");
})
.on("end", function () {
done();
});
});
it("should not send json if needEmitAll is false", function () {
return __awaiter(this, void 0, void 0, function () {
var data;
return __generator(this, function (_a) {
data = "a,b,c\n1,2,3\n4,5,6";
return [2 /*return*/, src_1.default({
needEmitAll: false
})
.fromString(data)
.then(function (d) {
assert_1.default(d.length === 0);
})];
});
});
});
it("should convert null to null object", function () {
return __awaiter(this, void 0, void 0, function () {
var data;
return __generator(this, function (_a) {
data = "a,b,c\nnull,2,3\n4,5,6";
return [2 /*return*/, src_1.default({
nullObject: true
})
.fromString(data)
.then(function (d) {
assert_1.default.equal(d[0].a, null);
})];
});
});
});
it("should process period properly", function () {
return __awaiter(this, void 0, void 0, function () {
var data;
return __generator(this, function (_a) {
data = "a..,b,c\n1,2,3\n4,5,6";
return [2 /*return*/, src_1.default({})
.fromString(data)
.then(function (d) {
assert_1.default.equal(d[0]["a.."], 1);
assert_1.default.equal(d[1]["a.."], 4);
})];
});
});
});
});
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiL1VzZXJzL2t4aWFuZy93b3JrL3Byb2plY3RzL2NzdjJqc29uL3Rlc3QvdGVzdENTVkNvbnZlcnRlcjMudHMiLCJzb3VyY2VzIjpbIi9Vc2Vycy9reGlhbmcvd29yay9wcm9qZWN0cy9jc3YyanNvbi90ZXN0L3Rlc3RDU1ZDb252ZXJ0ZXIzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7QUFBQSwrQ0FBeUI7QUFDekIsa0RBQTRCO0FBQzVCLElBQUksRUFBRSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUN2QiwrQkFBZ0M7QUFFaEMsSUFBTSxFQUFFLEdBQUcsZUFBTyxDQUFDLE1BQU0sRUFBRSxDQUFDO0FBQzVCLFFBQVEsQ0FBQyxtQkFBbUIsRUFBRTtJQUM1QixTQUFTLENBQUM7UUFDUixFQUFFLENBQUMsT0FBTyxFQUFFLENBQUM7SUFDZixDQUFDLENBQUMsQ0FBQztJQUNILEVBQUUsQ0FBQyxvRUFBb0UsRUFBRSxVQUFVLElBQUk7UUFDckYsSUFBSSxRQUFRLEdBQUcsU0FBUyxHQUFHLHNCQUFzQixDQUFDO1FBQ2xELElBQUksRUFBRSxHQUFHLEVBQUUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQztRQUN2QyxJQUFJLFlBQVksR0FBRyxhQUFHLENBQUMsRUFDdEIsQ0FBQyxDQUFDO1FBQ0gsSUFBSSxLQUFLLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsWUFBWSxDQUFDLFVBQVUsQ0FBQyxVQUFVLFVBQVU7WUFDMUMsZ0JBQU0sQ0FBQyxVQUFVLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxHQUFHLElBQUksQ0FBQyxDQUFDO1lBQ3hDLE9BQU8sVUFBVSxDQUFDO1FBQ3BCLENBQUMsQ0FBQyxDQUFBO1FBQ0YsWUFBWSxDQUFDLEVBQUUsQ0FBQyxNQUFNLEVBQUU7WUFDdEIsS0FBSyxFQUFFLENBQUM7UUFDVixDQUFDLENBQUMsQ0FBQztRQUNILFlBQVksQ0FBQyxJQUFJLENBQUM7WUFDaEIsZ0JBQU0sQ0FBQyxLQUFLLEtBQUssSUFBSSxDQUFDLENBQUM7WUFDdkIsSUFBSSxFQUFFLENBQUM7UUFDVCxDQUFDLENBQUMsQ0FBQztRQUNILEVBQUUsQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLENBQUM7SUFDeEIsQ0FBQyxDQUFDLENBQUM7SUFDSCxFQUFFLENBQUMsOENBQThDLEVBQUUsVUFBVSxJQUFJO1FBQy9ELGFBQUcsQ0FBQztZQUNGLFNBQVMsRUFBRSxJQUFJO1lBQ2YsU0FBUyxFQUFFO2dCQUNULFNBQVMsRUFBRSxRQUFRO2dCQUNuQixTQUFTLEVBQUUsVUFBVSxJQUFJLEVBQUUsSUFBSSxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsQ0FBQztvQkFDaEQsZ0JBQU0sQ0FBQyxLQUFLLENBQUMsSUFBSSxFQUFFLG1CQUFtQixDQUFDLENBQUM7b0JBQ3hDLGdCQUFNLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxTQUFTLENBQUM7d0JBQzNCLGdCQUFNLENBQUMsU0FBUyxDQUFDLENBQUM7b0JBQ3BCLGdCQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7b0JBQ1osZ0JBQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO29CQUNuQixPQUFPLGFBQWEsQ0FBQztnQkFDdkIsQ0FBQzthQUNGO1NBQ0YsQ0FBQzthQUNDLFFBQVEsQ0FBQyxTQUFTLEdBQUcsb0JBQW9CLENBQUM7YUFDMUMsU0FBUyxDQUFDLFVBQVUsSUFBSTtZQUN2QixnQkFBTSxDQUFDLEtBQUssQ0FBQyxPQUFPLElBQUksQ0FBQyxPQUFPLEVBQUUsUUFBUSxDQUFDLENBQUM7WUFDNUMsZ0JBQU0sQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxhQUFhLENBQUMsQ0FBQztZQUMxQyxnQkFBTSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDMUMsZ0JBQU0sQ0FBQyxXQUFXLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQzVDLENBQUMsQ0FBQzthQUNELEVBQUUsQ0FBQyxNQUFNLEVBQUU7WUFDVixJQUFJLEVBQUUsQ0FBQTtRQUNSLENBQUMsQ0FBQyxDQUFDO0lBQ1AsQ0FBQyxDQUFDLENBQUE7SUFDRixFQUFFLENBQUMsNkJBQTZCLEVBQUUsVUFBVSxJQUFJO1FBQzlDLGFBQUcsQ0FBQztZQUNGLEtBQUssRUFBRSxHQUFHO1lBQ1YsTUFBTSxFQUFFLEtBQUs7U0FDZCxDQUFDO2FBQ0MsUUFBUSxDQUFDLFNBQVMsR0FBRyxtQkFBbUIsQ0FBQzthQUN6QyxTQUFTLENBQUMsVUFBVSxHQUFHO1lBQ3RCLGdCQUFNLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxjQUFjLENBQUMsQ0FBQztRQUN2QyxDQUFDLENBQUM7YUFDRCxFQUFFLENBQUMsTUFBTSxFQUFFO1lBQ1YsSUFBSSxFQUFFLENBQUE7UUFDUixDQUFDLENBQUMsQ0FBQztJQUNQLENBQUMsQ0FBQyxDQUFBO0lBQ0YsRUFBRSxDQUFDLCtEQUErRCxFQUFFO1FBQ2xFLElBQUksTUFBTSxHQUFHLEtBQUssQ0FBQztRQUNuQixJQUFNLEVBQUUsR0FBRyxFQUFFLENBQUMsR0FBRyxDQUFDLFVBQUMsR0FBRztZQUNwQixnQkFBTSxDQUFDLEdBQUcsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxPQUFPLENBQUMscUJBQXFCLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQzdELENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxhQUFHLEVBQUU7YUFDVCxRQUFRLENBQUMsVUFBVSxDQUFDO2FBQ3BCLFNBQVMsQ0FBQyxVQUFVLEdBQUc7UUFFeEIsQ0FBQyxDQUFDO2FBQ0QsRUFBRSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUM7YUFDZixJQUFJLENBQUM7WUFDSixnQkFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQ2hCLENBQUMsRUFBRSxVQUFDLEdBQUc7WUFDTCxnQkFBTSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxDQUFDO1FBQ2hDLENBQUMsQ0FBQyxDQUFBO0lBRU4sQ0FBQyxDQUFDLENBQUE7SUFDRixFQUFFLENBQUMsMERBQTBELEVBQUU7UUFDN0QsT0FBTyxhQUFHLENBQUM7WUFDVCxjQUFjLEVBQUUsR0FBRztZQUNuQixhQUFhLEVBQUUsS0FBSztTQUNyQixDQUFDO2FBQ0MsVUFBVSxDQUFDLHFCQUVaLENBQUM7YUFDQSxTQUFTLENBQUMsVUFBQyxDQUFDO1lBQ1gsZ0JBQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDWixnQkFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQ2YsQ0FBQyx