diff --git a/CHANGELOG.md b/CHANGELOG.md index 7221eb9..02b6a01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 24.10.4 +- Added a new init time flag `salt` for request tampering protection (should be used in tandem with server options) + ## 24.10.3 - Added support for uploading user images by providing path to the local image using `picturePath` parameter in `user_details` method (non-bulk) - Reduced SDK log verbosity diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index bcfcd80..3ecdc93 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -40,6 +40,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums; * @param {number} [conf.session_update=60] - how often in seconds should session be extended * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch * @param {boolean} [conf.force_post=false] - force using post method for all requests + * @param {string} [conf.salt] - shared secret used to append checksum256 to outgoing requests * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request * @param {number} [conf.max_key_length=128] - maximum size of all string keys @@ -59,7 +60,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums; * }); */ function CountlyBulk(conf) { - var SDK_VERSION = "24.10.3"; + var SDK_VERSION = "24.10.4"; var SDK_NAME = "javascript_native_nodejs_bulk"; var empty_queue_callback = null; @@ -96,6 +97,7 @@ function CountlyBulk(conf) { conf.session_update = conf.session_update || 60; conf.max_events = conf.max_events || 100; conf.force_post = conf.force_post || false; + conf.salt = conf.salt || null; conf.persist_queue = conf.persist_queue || false; conf.http_options = conf.http_options || null; conf.maxKeyLength = conf.max_key_length || maxKeyLength; @@ -467,16 +469,12 @@ function CountlyBulk(conf) { } /** - * Convert JSON object to query params + * Convert JSON object to query params and append checksum when configured * @param {Object} params - object with url params * @returns {String} query string */ function prepareParams(params) { - var str = []; - for (var i in params) { - str.push(`${i}=${encodeURIComponent(params[i])}`); - } - return str.join("&"); + return cc.addChecksum(cc.serializeParams(params), conf.salt, false, false); } /** diff --git a/lib/countly-common.js b/lib/countly-common.js index f92023f..fe5140a 100644 --- a/lib/countly-common.js +++ b/lib/countly-common.js @@ -1,6 +1,8 @@ /** * main common functionalities will go in here */ +var crypto = require("crypto"); + var cc = { // debug value from Countly @@ -135,6 +137,64 @@ var cc = { } return ob; }, + /** + * Convert params object to URL encoded query parameter string + * @param {Object} params - object with query parameters + * @returns {String} URL encoded query string + */ + serializeParams: function serializeParams(params) { + var str = []; + var keys = Object.keys(params || {}); + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + str.push(`${key}=${encodeURIComponent(params[key])}`); + } + return str.join("&"); + }, + /** + * Calculate the SHA-256 checksum for provided request data and salt + * @param {String} data - serialized request data + * @param {String} salt - developer provided shared secret + * @param {Boolean} decodeBeforeHash - if true decodes serialized data before hashing + * @param {Boolean} uppercase - if true returns uppercase hex + * @returns {String} checksum in hex format + */ + calculateChecksum: function calculateChecksum(data, salt, decodeBeforeHash, uppercase) { + var checksumData = data || ""; + if (decodeBeforeHash) { + try { + checksumData = decodeURIComponent(checksumData); + } + catch (e) { + this.log(this.logLevelEnums.WARNING, `calculateChecksum, Failed to decode request data before hashing: [${e}]`); + } + } + var hash = crypto.createHash("sha256"); + hash.update(`${checksumData}${salt}`); + var checksum = hash.digest("hex"); + if (uppercase) { + return checksum.toUpperCase(); + } + return checksum; + }, + /** + * Append checksum256 to serialized request data when salt is configured + * @param {String} data - serialized request data + * @param {String} salt - developer provided shared secret + * @param {Boolean} decodeBeforeHash - if true decodes serialized data before hashing + * @param {Boolean} uppercase - if true appends uppercase hex + * @returns {String} serialized request data with checksum when configured + */ + addChecksum: function addChecksum(data, salt, decodeBeforeHash, uppercase) { + if (!salt) { + return data; + } + var checksum = this.calculateChecksum(data, salt, decodeBeforeHash, uppercase); + if (!data) { + return `checksum256=${checksum}`; + } + return `${data}&checksum256=${checksum}`; + }, /** * Removing trailing slashes * @memberof Countly._internals diff --git a/lib/countly.js b/lib/countly.js index 0963e70..f0a0181 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -35,7 +35,7 @@ Countly.StorageTypes = cc.storageTypeEnums; Countly.DeviceIdType = cc.deviceIdTypeEnums; Countly.Bulk = Bulk; (function() { - var SDK_VERSION = "24.10.3"; + var SDK_VERSION = "24.10.4"; var SDK_NAME = "javascript_native_nodejs"; var inited = false; @@ -103,6 +103,7 @@ Countly.Bulk = Bulk; * @param {number} [conf.session_update=60] - how often in seconds should session be extended * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch * @param {boolean} [conf.force_post=false] - force using post method for all requests + * @param {string} [conf.salt] - shared secret used to append checksum256 to outgoing requests * @param {boolean} [conf.clear_stored_device_id=false] - set it to true if you want to erase the stored device ID * @param {boolean} [conf.test_mode=false] - set it to true if you want to initiate test_mode * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc @@ -162,6 +163,7 @@ Countly.Bulk = Bulk; Countly.city = conf.city || Countly.city || null; Countly.ip_address = conf.ip_address || Countly.ip_address || null; Countly.force_post = conf.force_post || Countly.force_post || false; + Countly.salt = conf.salt || Countly.salt || null; Countly.require_consent = conf.require_consent || Countly.require_consent || false; Countly.remote_config = conf.remote_config || Countly.remote_config || false; Countly.http_options = conf.http_options || Countly.http_options || null; @@ -215,6 +217,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.DEBUG, `init, IP address: [${Countly.ip_address}].`); } cc.log(cc.logLevelEnums.DEBUG, `init, Force POST requests: [${Countly.force_post}].`); + cc.log(cc.logLevelEnums.DEBUG, `init, Salt is configured: [${!!Countly.salt}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Storage path: [${CountlyStorage.getStoragePath()}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Require consent: [${Countly.require_consent}].`); if (Countly.remote_config) { @@ -353,6 +356,7 @@ Countly.Bulk = Bulk; Countly.city = undefined; Countly.ip_address = undefined; Countly.force_post = undefined; + Countly.salt = undefined; Countly.require_consent = undefined; Countly.http_options = undefined; CountlyStorage.resetStorage(); @@ -1670,17 +1674,36 @@ Countly.Bulk = Bulk; var boundary = `FormBoundary${Math.random().toString(16).slice(2)}`; var bodyParts = []; + var uploadParams = {}; for (var p in params) { - if (typeof params[p] !== "undefined" && p !== 'picturePath') { - var value = params[p]; + if (Object.prototype.hasOwnProperty.call(params, p) && typeof params[p] !== "undefined" && p !== "picturePath") { + uploadParams[p] = params[p]; + } + } + + var uploadChecksum = null; + if (Countly.salt) { + uploadChecksum = cc.calculateChecksum(cc.serializeParams(uploadParams), Countly.salt, true); + } + + for (var param in uploadParams) { + if (Object.prototype.hasOwnProperty.call(uploadParams, param)) { + var value = uploadParams[param]; bodyParts.push(Buffer.from(`--${boundary}\r\n`)); - bodyParts.push(Buffer.from(`Content-Disposition: form-data; name="${p}"\r\n\r\n`)); + bodyParts.push(Buffer.from(`Content-Disposition: form-data; name="${param}"\r\n\r\n`)); bodyParts.push(Buffer.from(String(value))); bodyParts.push(Buffer.from('\r\n')); } } + if (uploadChecksum) { + bodyParts.push(Buffer.from(`--${boundary}\r\n`)); + bodyParts.push(Buffer.from('Content-Disposition: form-data; name="checksum256"\r\n\r\n')); + bodyParts.push(Buffer.from(uploadChecksum)); + bodyParts.push(Buffer.from('\r\n')); + } + bodyParts.push(Buffer.from(`--${boundary}\r\n`)); bodyParts.push(Buffer.from(`Content-Disposition: form-data; name="user_picture"; filename="${fileName}"\r\n`)); bodyParts.push(Buffer.from(`Content-Type: ${contentType}\r\n\r\n`)); @@ -1779,16 +1802,14 @@ Countly.Bulk = Bulk; } /** - * Convert JSON object to query params + * Convert JSON object to query params and append checksum when configured * @param {Object} params - object with url params + * @param {Boolean} decodeBeforeHash - if true request data is URL-decoded before hashing * @returns {String} query string */ - function prepareParams(params) { - var str = []; - for (var i in params) { - str.push(`${i}=${encodeURIComponent(params[i])}`); - } - return str.join("&"); + function prepareParams(params, decodeBeforeHash) { + var data = cc.serializeParams(params); + return cc.addChecksum(data, Countly.salt, decodeBeforeHash, false); } /** diff --git a/package-lock.json b/package-lock.json index 7a432b0..0a497a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "countly-sdk-nodejs", - "version": "24.10.3", + "version": "24.10.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "countly-sdk-nodejs", - "version": "24.10.3", + "version": "24.10.4", "license": "MIT", "devDependencies": { "docdash": "2.0.2", diff --git a/package.json b/package.json index 578eccc..20c8138 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "countly-sdk-nodejs", - "version": "24.10.3", + "version": "24.10.4", "description": "Countly NodeJS SDK", "main": "lib/countly.js", "directories": { diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 996f971..361771e 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -38,17 +38,40 @@ const sWait = 50; const mWait = 3000; const lWait = 10000; +function readStoredValue(storageKey, destination, fileKey, fallbackValue) { + try { + const storedValue = CountlyStorage.storeGet(storageKey, undefined); + if (typeof storedValue !== "undefined") { + return storedValue; + } + } + catch (error) { + // Ignore storage access errors and fall back to direct file reads. + } + + try { + const fileData = JSON.parse(fs.readFileSync(destination, "utf-8")); + if (fileData && typeof fileData[fileKey] !== "undefined") { + return fileData[fileKey]; + } + } + catch (error) { + // Ignore incomplete or missing file contents and return the fallback value. + } + + return fallbackValue; +} + // parsing event queue function readEventQueue(givenPath = null, isBulk = false) { var destination = DIR_CLY_event; if (givenPath !== null) { destination = givenPath; } - var a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_event; if (isBulk) { - a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_bulk_event; + return readStoredValue("cly_bulk_event", destination, "cly_bulk_event", {}); } - return a; + return readStoredValue("cly_event", destination, "cly_event", []); } // parsing request queue function readRequestQueue(customPath = false, isBulk = false, isMemory = false) { @@ -56,17 +79,13 @@ function readRequestQueue(customPath = false, isBulk = false, isMemory = false) if (customPath) { destination = DIR_Test_request; } - var a; if (isBulk) { - a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_req_queue; + return readStoredValue("cly_req_queue", destination, "cly_req_queue", []); } if (isMemory) { - a = CountlyStorage.storeGet("cly_queue"); - } - else { - a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_queue; + return CountlyStorage.storeGet("cly_queue", []); } - return a; + return readStoredValue("cly_queue", destination, "cly_queue", []); } function doesFileStoragePathsExist(callback, isBulk = false, testPath = false) { var paths = [DIR_CLY_ID, DIR_CLY_ID_type, DIR_CLY_event, DIR_CLY_request]; diff --git a/test/tests_salt.js b/test/tests_salt.js new file mode 100644 index 0000000..5a7db3a --- /dev/null +++ b/test/tests_salt.js @@ -0,0 +1,306 @@ +const assert = require("assert"); +const crypto = require("crypto"); +const fs = require("fs"); +const http = require("http"); +const path = require("path"); +const Countly = require("../lib/countly"); +const CountlyBulk = require("../lib/countly-bulk"); +const cc = require("../lib/countly-common"); +const hp = require("./helpers/helper_functions"); + +const salt = "salt"; + +function sha256(data) { + return crypto.createHash("sha256").update(data).digest("hex"); +} + +function getRequestData(request) { + if (request.method === "GET") { + const queryIndex = request.url.indexOf("?"); + return queryIndex === -1 ? "" : request.url.substring(queryIndex + 1); + } + return request.body.toString("utf8"); +} + +function splitChecksum(data) { + if (data.indexOf("checksum256=") === -1) { + return { data: data, checksum: null }; + } + + if (data.indexOf("&checksum256=") !== -1) { + const markerIndex = data.lastIndexOf("&checksum256="); + return { + data: data.substring(0, markerIndex), + checksum: data.substring(markerIndex + "&checksum256=".length), + }; + } + + return { + data: "", + checksum: data.substring("checksum256=".length), + }; +} + +function parseMultipartFields(contentType, bodyBuffer) { + const boundaryMatch = /boundary=([^;]+)/.exec(contentType || ""); + assert.ok(boundaryMatch, "Expected multipart boundary"); + + const boundary = `--${boundaryMatch[1]}`; + const body = bodyBuffer.toString("utf8"); + const parts = body.split(boundary); + const fields = []; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (part && part !== "--\r\n" && part !== "--") { + const nameMatch = /name="([^"]+)"/.exec(part); + const valueStart = part.indexOf("\r\n\r\n"); + + if (nameMatch && valueStart !== -1) { + let value = part.substring(valueStart + 4); + if (value.endsWith("\r\n")) { + value = value.substring(0, value.length - 2); + } + if (value.endsWith("--")) { + value = value.substring(0, value.length - 2); + } + + fields.push({ name: nameMatch[1], value: value }); + } + } + } + + return fields; +} + +function closeServer(server) { + return new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +async function startServer() { + const state = { + requests: [], + waiters: [], + }; + + const server = http.createServer((req, res) => { + const chunks = []; + req.on("data", (chunk) => { + chunks.push(chunk); + }); + req.on("end", () => { + const request = { + method: req.method, + url: req.url, + headers: req.headers, + body: Buffer.concat(chunks), + }; + + state.requests.push(request); + if (state.waiters.length > 0) { + const waiter = state.waiters.shift(); + waiter(request); + } + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end('{"result":"Success"}'); + }); + }); + + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + + const address = server.address(); + + return { + server: server, + state: state, + url: `http://127.0.0.1:${address.port}`, + }; +} + +function waitForNextRequest(state, timeoutMs = 3000) { + return new Promise((resolve, reject) => { + if (state.requests.length > 0) { + resolve(state.requests.shift()); + return; + } + + const timeoutId = setTimeout(() => { + reject(new Error("Timed out waiting for request")); + }, timeoutMs); + + state.waiters.push((request) => { + clearTimeout(timeoutId); + resolve(request); + }); + }); +} + +function delay(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +describe("Salt helper tests", () => { + it("serializes params in insertion order and produces lowercase checksums", () => { + const encodedData = cc.serializeParams({ zeta: 1, alpha: 2 }); + assert.equal(encodedData, "zeta=1&alpha=2"); + assert.equal(cc.calculateChecksum(encodedData, salt), sha256(encodedData + salt)); + assert.equal(cc.addChecksum(encodedData, salt, false), `${encodedData}&checksum256=${sha256(encodedData + salt)}`); + }); + + it("URL-decodes request data before hashing when requested", () => { + const encodedData = "user_details=%7B%22name%22%3A%22A%20%26%20B%22%7D"; + assert.equal(cc.calculateChecksum(encodedData, salt, true), sha256(`${decodeURIComponent(encodedData)}${salt}`)); + }); +}); + +describe("Salt integration tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + Countly.halt(false); + }); + + it("does not append checksum256 when salt is not configured", async() => { + const serverInfo = await startServer(); + + try { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: serverInfo.url, + interval: 10, + }); + + Countly.begin_session(true); + + const request = await waitForNextRequest(serverInfo.state); + const data = getRequestData(request); + + assert.equal(data.indexOf("checksum256=") !== -1, false); + await delay(50); + } + finally { + Countly.halt(true); + await closeServer(serverInfo.server); + } + }); + + it("appends a lowercase checksum using natural parameter order", async() => { + const serverInfo = await startServer(); + + try { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: serverInfo.url, + interval: 10, + salt: salt, + }); + + Countly.request({ + zeta: "1", + alpha: "2", + app_key: "YOUR_APP_KEY", + device_id: "salt-device", + }); + + const request = await waitForNextRequest(serverInfo.state); + const requestData = getRequestData(request); + const checksumData = splitChecksum(requestData); + + assert.ok(checksumData.checksum); + assert.match(checksumData.checksum, /^[a-f0-9]{64}$/); + assert.ok(checksumData.data.indexOf("zeta=1") < checksumData.data.indexOf("alpha=2")); + assert.equal(checksumData.checksum, sha256(`${checksumData.data}${salt}`)); + await delay(50); + } + finally { + Countly.halt(true); + await closeServer(serverInfo.server); + } + }); + + it("URL-decodes upload request data before hashing for picture uploads", async() => { + const serverInfo = await startServer(); + const imagePath = path.join(__dirname, "salt-upload-test.png"); + fs.writeFileSync(imagePath, Buffer.from("fake-image-binary")); + + try { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: serverInfo.url, + interval: 10, + salt: salt, + }); + + Countly.user_details({ + name: "A & B", + picturePath: imagePath, + }); + + const request = await waitForNextRequest(serverInfo.state); + const fields = parseMultipartFields(request.headers["content-type"], request.body); + const checksumField = fields.find((field) => field.name === "checksum256"); + const dataFields = fields.filter((field) => field.name !== "checksum256" && field.name !== "user_picture"); + const rawRequestData = dataFields.map((field) => `${field.name}=${field.value}`).join("&"); + + assert.equal(request.method, "POST"); + assert.ok(checksumField); + assert.ok(dataFields.some((field) => field.name === "user_details" && field.value.indexOf("A & B") !== -1)); + assert.equal(checksumField.value, sha256(`${rawRequestData}${salt}`)); + await delay(50); + } + finally { + Countly.halt(true); + if (fs.existsSync(imagePath)) { + fs.unlinkSync(imagePath); + } + await closeServer(serverInfo.server); + } + }); + + it("appends checksum256 to bulk requests when salt is configured", async() => { + const serverInfo = await startServer(); + const bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: serverInfo.url, + interval: 10, + bulk_size: 1, + salt: salt, + }); + + try { + bulk.add_request({ + zeta: "1", + alpha: "2", + device_id: "bulk-device", + }); + bulk.start(); + + const request = await waitForNextRequest(serverInfo.state); + const requestData = getRequestData(request); + const checksumData = splitChecksum(requestData); + + assert.ok(checksumData.checksum); + assert.match(checksumData.checksum, /^[a-f0-9]{64}$/); + assert.ok(checksumData.data.indexOf("app_key=YOUR_APP_KEY") !== -1); + assert.ok(checksumData.data.indexOf("requests=%5B") !== -1); + assert.equal(checksumData.checksum, sha256(`${checksumData.data}${salt}`)); + await delay(50); + } + finally { + bulk.stop(); + await closeServer(serverInfo.server); + } + }); +}); \ No newline at end of file