From 56c736191a0a87976b589f7eb4666e9b9974dd17 Mon Sep 17 00:00:00 2001 From: angerhang <514140809@qq.com> Date: Thu, 1 Dec 2022 19:43:14 +0000 Subject: [PATCH 1/2] * Include lux extraction from ax3 to epoch file * Add dummy light variable to device readers without light --- accelerometer/java/ActigraphReader.java | 12 ++++++---- accelerometer/java/AxivityReader.java | 4 ++-- accelerometer/java/CsvReader.java | 25 +++++++++++++------- accelerometer/java/EpochWriter.java | 31 ++++++++++++++++--------- accelerometer/java/GENEActivReader.java | 3 ++- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/accelerometer/java/ActigraphReader.java b/accelerometer/java/ActigraphReader.java index ca339bb0..5cfb0c6f 100644 --- a/accelerometer/java/ActigraphReader.java +++ b/accelerometer/java/ActigraphReader.java @@ -379,8 +379,9 @@ private static void readG3TXV1EpochPairs( double y = twoSamples[4-twoSampleCounter*3]; double z = twoSamples[5-twoSampleCounter*3]; double temp = 1.0d; // don't know temp yet + double light = 1.0d; // don't know light yet time = getTrueUnixTime(time, infoTimeShift); - epochWriter.newValues(time, x, y, z, temp, errCounter); + epochWriter.newValues(time, x, y, z, temp, light, errCounter); samples += 1; } @@ -449,10 +450,11 @@ private static void readG3TXV1EpochPairs( logger.log(Level.FINER, "x y z: " + sample[1] + " " + sample[0] + " " + sample[2]); double temp = 1.0d; // don't know temp yet + double light = 1.0d; // don't know light yet samples += 1; long myTime = Math.round((1000d*samples)/sampleFreq) + firstSampleTime*1000; // in Miliseconds myTime = getTrueUnixTime(myTime, infoTimeShift); - epochWriter.newValues(myTime, sample[1], sample[0], sample[2], temp, errCounter); + epochWriter.newValues(myTime, sample[1], sample[0], sample[2], temp, light, errCounter); } } catch (IOException ex) { ex.printStackTrace(System.err); @@ -517,6 +519,8 @@ public static long getTrueUnixTime(long myTime, String infoTimeShift) { } double temp = 1.0d; // don't know temp yet + double light = 1.0d; // don't know light yet + samples += 1; long myTime = Math.round((1000d*samples)/sampleFreq) + firstSampleTime*1000; // in Miliseconds @@ -525,7 +529,7 @@ public static long getTrueUnixTime(long myTime, String infoTimeShift) { logger.log(Level.FINER, "i: " + i + "\nx y z: " + sample[0] + " " + sample[1] + " " + sample[2] + "\nTime:" + myTime); epochWriter.newValues(myTime, - sample[0], sample[1], sample[2], temp, errCounter); + sample[0], sample[1], sample[2], temp, light, errCounter); } } catch (IOException ex) { ex.printStackTrace(System.err); @@ -747,4 +751,4 @@ private static int getGT3XVersion(final ZipFile zip) throws IOException { } -} \ No newline at end of file +} diff --git a/accelerometer/java/AxivityReader.java b/accelerometer/java/AxivityReader.java index 2032e0ff..8e518d6a 100644 --- a/accelerometer/java/AxivityReader.java +++ b/accelerometer/java/AxivityReader.java @@ -160,7 +160,7 @@ private static void readCwaBuffer(ByteBuffer buf, try { // read block header items long blockTimestamp = getUnsignedInt(buf, 14); - int light = getUnsignedShort(buf, 18); + double light = getUnsignedShort(buf, 18) & 0x3ff; double temperature = (getUnsignedShort(buf, 20) * 150.0 - 20500) / 1000; short rateCode = (short) (buf.get(24) & 0xff); short numAxesBPS = (short) (buf.get(25) & 0xff); @@ -318,7 +318,7 @@ private static void readCwaBuffer(ByteBuffer buf, z = zRaw / 256.0; t = zonedWithDSTCorrection(blockTime).toInstant().toEpochMilli(); - epochWriter.newValues(t, x, y, z, temperature, errCounter); + epochWriter.newValues(t, x, y, z, temperature, light, errCounter); } } catch (Exception excep) { diff --git a/accelerometer/java/CsvReader.java b/accelerometer/java/CsvReader.java index 972265f0..533b7269 100644 --- a/accelerometer/java/CsvReader.java +++ b/accelerometer/java/CsvReader.java @@ -16,12 +16,13 @@ public class CsvReader extends DeviceReader { private static int NO_TEMPERATURE = -99; + private static int NO_LIGHT = -99; public static void readCSVEpochs( String accFile, EpochWriter epochWriter, int csvStartRow, - List csvTimeXYZTempColsIndex, + List csvTimeXYZTempLightColsIndex, DateTimeFormatter csvTimeFormat, Boolean verbose) { @@ -40,13 +41,17 @@ public static void readCSVEpochs( String line = ""; int lineNumber = 0; String csvSplitBy = ","; - int timeCol = csvTimeXYZTempColsIndex.get(0); - int xCol = csvTimeXYZTempColsIndex.get(1); - int yCol = csvTimeXYZTempColsIndex.get(2); - int zCol = csvTimeXYZTempColsIndex.get(3); + int timeCol = csvTimeXYZTempLightColsIndex.get(0); + int xCol = csvTimeXYZTempLightColsIndex.get(1); + int yCol = csvTimeXYZTempLightColsIndex.get(2); + int zCol = csvTimeXYZTempLightColsIndex.get(3); int temperatureCol = NO_TEMPERATURE; - if (csvTimeXYZTempColsIndex.size() == 5){ - temperatureCol = csvTimeXYZTempColsIndex.get(4); + int lightCol = NO_LIGHT; + if (csvTimeXYZTempLightColsIndex.size() > 4){ + temperatureCol = csvTimeXYZTempLightColsIndex.get(4); + if (csvTimeXYZTempLightColsIndex.size() == 6) { + lightCol = csvTimeXYZTempLightColsIndex.get(5); + } } String[] cols; @@ -55,6 +60,7 @@ public static void readCSVEpochs( double y; double z; double temperature = 0; + double light = 0; while (true) { line = accReader.readLine(); @@ -78,7 +84,10 @@ public static void readCSVEpochs( if (temperatureCol != NO_TEMPERATURE){ temperature = Double.parseDouble(cols[temperatureCol]); } - epochWriter.newValues(time, x, y, z, temperature, new int[] {0}); + if (lightCol != NO_LIGHT) { + light = Double.parseDouble(cols[lightCol]); + } + epochWriter.newValues(time, x, y, z, temperature, light, new int[] {0}); } } catch (Exception excep) { diff --git a/accelerometer/java/EpochWriter.java b/accelerometer/java/EpochWriter.java index 18bab270..3571f86b 100644 --- a/accelerometer/java/EpochWriter.java +++ b/accelerometer/java/EpochWriter.java @@ -31,11 +31,12 @@ public class EpochWriter { private List xVals = new ArrayList(); private List yVals = new ArrayList(); private List zVals = new ArrayList(); + private List lightVals = new ArrayList(); private List temperatureVals = new ArrayList(); private final int minSamplesForEpoch = 10; private long prevTimeVal = -1; - private double[] prevXYZT = { -1, -1, -1, -1 }; // x/y/z/temp at prevTimeVal + private double[] prevXYZTL = { -1, -1, -1, -1, -1}; // x/y/z/temp/light at prevTimeVal private boolean edgeInterpolation = true; // parameters to be initialised @@ -117,7 +118,7 @@ public EpochWriter( String epochHeader = "time"; epochHeader += "," + AccStats.getStatsHeader(getFeatures); - epochHeader += ",temp,samples"; + epochHeader += ",temp,light,samples"; epochHeader += ",dataErrors,clipsBeforeCalibr,clipsAfterCalibr,rawSamples"; writeLine(epochFileWriter, epochHeader); @@ -136,6 +137,7 @@ public boolean newValues( double y, double z, double temperature, + double light, int[] errCounter) { if (startTime!=UNUSED_DATE && timeminSamplesForEpoch) { writeEpochSummary(millisToZonedDateTime(epochStartTime), timeVals, // writeEpochSummary(millisToInstant(epochStartTime), timeVals, - xVals, yVals, zVals, temperatureVals, errCounter); + xVals, yVals, zVals, temperatureVals, lightVals, errCounter); } else { System.err.println("not enough samples for an epoch.. discarding " + timeVals.size()+" samples"); @@ -185,6 +187,7 @@ public boolean newValues( yVals.clear(); zVals.clear(); temperatureVals.clear(); + lightVals.clear(); errCounter[0] = 0; } // epoch times must be at regular (epochPeriod) intervals, so move forward @@ -204,10 +207,11 @@ public boolean newValues( yVals.add(y); zVals.add(z); temperatureVals.add(temperature); + lightVals.add(light); } writeEpochSummary(millisToZonedDateTime(epochStartTime), timeVals, // writeEpochSummary(millisToInstant(epochStartTime), timeVals, - xVals, yVals, zVals, temperatureVals, errCounter); + xVals, yVals, zVals, temperatureVals, lightVals, errCounter); epochStartTime = epochStartTime + epochPeriod * 1000; @@ -215,10 +219,11 @@ public boolean newValues( // this code adds the first sample of the previous epoch so we //can correctly interpolate to the edges timeVals.add(prevTimeVal - epochStartTime); - xVals.add(prevXYZT[0]); - yVals.add(prevXYZT[1]); - zVals.add(prevXYZT[2]); - temperatureVals.add(prevXYZT[3]); + xVals.add(prevXYZTL[0]); + yVals.add(prevXYZTL[1]); + zVals.add(prevXYZTL[2]); + temperatureVals.add(prevXYZTL[3]); + lightVals.add(prevXYZTL[4]); } } if (endTime!=UNUSED_DATE && time>endTime) { @@ -240,7 +245,7 @@ public boolean newValues( zVals.add(z); temperatureVals.add(temperature); prevTimeVal = time; - prevXYZT = new double[]{x, y, z, temperature}; + prevXYZTL = new double[]{x, y, z, temperature, light}; return true; } @@ -262,6 +267,7 @@ private void writeEpochSummary( List yVals, List zVals, List temperatureVals, + List lightVals, int[] errCounter) { int[] clipsCounter = new int[] { 0, 0 }; // before, after (calibration) @@ -269,13 +275,14 @@ private void writeEpochSummary( double y; double z; double temp; + for (int i = 0; i < xVals.size(); i++) { Boolean isClipped = false; x = xVals.get(i); y = yVals.get(i); z = zVals.get(i); - temp = temperatureVals.get(i); - // temp + temp = temperatureVals.get(i); // temp + // check if any pre-calibration clipping present //use >= range as it's clipped here if (Math.abs(x) >= range || Math.abs(y) >= range || Math.abs(z) >= range) { @@ -370,6 +377,7 @@ private void writeEpochSummary( // write housekeeping stats epochSummary += "," + DF2.format(AccStats.mean(temperatureVals)); + epochSummary += ',' + DF2.format(AccStats.mean(lightVals)); epochSummary += "," + xResampled.length + "," + errCounter[0]; epochSummary += "," + clipsCounter[0] + "," + clipsCounter[1]; epochSummary += "," + timeVals.size(); @@ -387,6 +395,7 @@ private void writeEpochSummary( yVals.clear(); zVals.clear(); temperatureVals.clear(); + lightVals.clear(); errCounter[0] = 0; } diff --git a/accelerometer/java/GENEActivReader.java b/accelerometer/java/GENEActivReader.java index f2e3b6ff..68fb4c43 100644 --- a/accelerometer/java/GENEActivReader.java +++ b/accelerometer/java/GENEActivReader.java @@ -111,7 +111,8 @@ public static void readGeneaEpochs( z = (zRaw * 100.0d - mfrOffset[2]) / mfrGain[2]; t = zonedWithDSTCorrection(blockTime).toInstant().toEpochMilli(); - epochWriter.newValues(t, x, y, z, temperature, errCounter); + double light = 1.0d; // don't know light yet + epochWriter.newValues(t, x, y, z, temperature, light, errCounter); hexPosition += 12; blockTime = blockTime.plusNanos(secs2Nanos(1.0 / sampleFreq)); From 01f54bc38023cf9905bd511f1857019747c7f50d Mon Sep 17 00:00:00 2001 From: angerhang <514140809@qq.com> Date: Fri, 2 Dec 2022 14:11:48 +0000 Subject: [PATCH 2/2] Compute summary stats from ax3 light epoch data --- accelerometer/java/EpochWriter.java | 2 +- accelerometer/summarisation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/accelerometer/java/EpochWriter.java b/accelerometer/java/EpochWriter.java index 3571f86b..1d789dac 100644 --- a/accelerometer/java/EpochWriter.java +++ b/accelerometer/java/EpochWriter.java @@ -118,7 +118,7 @@ public EpochWriter( String epochHeader = "time"; epochHeader += "," + AccStats.getStatsHeader(getFeatures); - epochHeader += ",temp,light,samples"; + epochHeader += ",temp,ax3_light,samples"; epochHeader += ",dataErrors,clipsBeforeCalibr,clipsAfterCalibr,rawSamples"; writeLine(epochFileWriter, epochHeader); diff --git a/accelerometer/summarisation.py b/accelerometer/summarisation.py index 39c7bbf5..20dd2799 100644 --- a/accelerometer/summarisation.py +++ b/accelerometer/summarisation.py @@ -352,7 +352,7 @@ def writeMovementSummaries(data, labels, summary): # noqa: C901 # Hours of activity for each recorded day epochInHours = pd.Timedelta(freq).total_seconds() / 3600 - cols = ['wearTime', 'cutPointMVPA', 'cutPointVPA'] + labels + cols = ['wearTime', 'cutPointMVPA', 'cutPointVPA', 'ax3_light'] + labels dailyStats = ( data[cols].astype('float') .groupby(data.index.date) @@ -367,7 +367,7 @@ def writeMovementSummaries(data, labels, summary): # noqa: C901 # In the following, we resample, pad and impute the data so that we have a # multiple of 24h for the stats calculations tStart, tEnd = data.index[0], data.index[-1] - cols = ['acc', 'wearTime', 'cutPointMVPA', 'cutPointVPA'] + labels + cols = ['acc', 'wearTime', 'cutPointMVPA', 'cutPointVPA', 'ax3_light'] + labels if 'MET' in data.columns: cols.append('MET') data = imputeMissing(data[cols].astype('float'))