.
--- a/RDI_PD0_IO.pl
+++ b/RDI_PD0_IO.pl
@@ -1,9 +1,9 @@
#======================================================================
-# R D I _ P D 0 _ I O . P L
+# / D A T A / S R C / O C E A N O G R A P H Y / A D C P _ T O O L S / R D I _ B B _ R E A D . P L
# doc: Sat Jan 18 14:54:43 2003
-# dlm: Wed Jun 26 09:27:46 2019
+# dlm: Thu Feb 27 10:28:29 2020
# (c) 2003 A.M. Thurnherr
-# uE-Info: 645 39 NIL 0 0 72 2 2 4 NIL ofnI
+# uE-Info: 716 1 NIL 0 0 72 0 2 4 NIL ofnI
#======================================================================
# Read RDI PD0 binary data files (*.[0-9][0-9][0-9])
@@ -115,6 +115,9 @@
# Jun 12, 2018: - BUG: IMPed files did not pass the garbage detection
# Jun 13, 2019: - adapted reading routines to RTI files (free order of data types)
# - removed old BT_PRESENT code
+# Jun 28, 2019: - renamed SECONDS to SECOND for consistency
+# Jun 30, 2019: - added dirty flag to prevent bad PD0 patching
+# Feb 13, 2020: - added support for $readDataProgress
# FIRMWARE VERSIONS:
# It appears that different firmware versions generate different file
@@ -292,7 +295,7 @@
# TIME string HH:MM:SS.hh
# HOUR scalar 0--23
# MINUTE scalar 0--59
-# SECONDS scalar 0--59.99
+# SECOND scalar 0--59.99
# UNIX_TIME scalar 0--?
# SECNO scalar 0--? (number of seconds since daystart)
# DAYNO double fractional day number since start of current year (1.0 is midnight Jan 1st)
@@ -452,8 +455,10 @@
#--------------------
my($skipped) = goto_next_ens(\*WBRF,1);
- printf(STDERR "WARNING: %d bytes of initial garbage\n",$skipped)
- if ($skipped > 0);
+ if ($skipped > 0) {
+ $RDI_PD0_IO::File_Dirty = 1;
+ printf(STDERR "WARNING: %d bytes of initial garbage\n",$skipped);
+ }
sysread(WBRF,$buf,6) == 6 || return undef;
($hid,$did,$dta->{ENSEMBLE_BYTES},$dummy,$dta->{NUMBER_OF_DATA_TYPES})
@@ -681,6 +686,8 @@
# - read ensembles
# - read all ensembles unless first_ens and last_ens are given
# - read all bins unless last_bin is given
+# - if global var $readDataProgress > 0, a . is printed
+# every $readaDataProgress ensembles
#----------------------------------------------------------------------
sub readData(@)
@@ -706,6 +713,8 @@
sysseek(WBRF,0,0) || die("$WBRcfn: $!");
ENSEMBLE:
for ($ens=0; 1; $ens++) {
+# die unless defined($global::readDataProgress);
+ print(STDERR '.') if ($global::readDataProgress>0 && ($ens%$global::readDataProgress)==0);
$start_ens = goto_next_ens(\*WBRF);
last unless defined($start_ens);
@@ -742,6 +751,7 @@
}
if (defined($ens_length) && ($el != $ens_length)) {
+ $RDI_PD0_IO::File_Dirty = 1;
print(STDERR "WARNING (RDI_PD0_IO): ensemble ${$E}[$#{$E}]->{NUMBER} skipped (unexpected length)\n");
pop(@{$E});
$ens--;
@@ -752,7 +762,14 @@
## printf(STDERR "$WBRcfn: WARNING: unexpected number of data types (%d, ens=$ens)\n",$ndt),last
## unless ($ndt == 6 || $ndt == 7);
- sysread(WBRF,$buf,2*$ndt) == 2*$ndt || die("$WBRcfn: $!");
+ my($nread) = sysread(WBRF,$buf,2*$ndt); # 2019 EPR test
+ if ($nread != 2*$ndt) {
+ printf(STDERR "$WBRcfn: WARNING: expected to read %d bytes, got only %d in ensemble %d\n",
+ 2*$ndt,$nread,${$E}[$ens]->{NUMBER});
+ last;
+ }
+
+
@WBRofs = unpack("v$ndt",$buf);
$fixed_leader_bytes = $WBRofs[1] - $WBRofs[0];
# print(STDERR "@WBRofs\n");
@@ -800,8 +817,8 @@
sysread(WBRF,$buf,7) == 7 || die("$WBRcfn: $!"); # always read pre-Y2K clock
(${$E}[$ens]->{YEAR},${$E}[$ens]->{MONTH},
${$E}[$ens]->{DAY},${$E}[$ens]->{HOUR},${$E}[$ens]->{MINUTE},
- ${$E}[$ens]->{SECONDS},$B4) = unpack('CCCCCCC',$buf);
- ${$E}[$ens]->{SECONDS} += $B4/100;
+ ${$E}[$ens]->{SECOND},$B4) = unpack('CCCCCCC',$buf);
+ ${$E}[$ens]->{SECOND} += $B4/100;
${$E}[$ens]->{YEAR} += (${$E}[$ens]->{YEAR} > 80) ? 1900 : 2000;
# } else {
# sysseek(WBRF,7,1) || die("$WBRcfn: $!"); # use Y2K RTC instead
@@ -812,6 +829,7 @@
for (my($i)=$ens; $i>0; $i--) { # check for duplicate ens; e.g. 2018 S4P 24UL
if (${$E}[$i]->{NUMBER} == $ensNo) {
+ $RDI_PD0_IO::File_Dirty = 1;
print(STDERR "WARNING (RDI_PD0_IO): duplicate ensemble $ensNo skipped\n");
pop(@{$E});
$ens--;
@@ -868,13 +886,13 @@
$dummy,${$E}[$ens]->{PRESSURE},${$E}[$ens]->{PRESSURE_STDDEV},
$dummy,${$E}[$ens]->{YEAR},$B3,${$E}[$ens]->{MONTH},
${$E}[$ens]->{DAY},${$E}[$ens]->{HOUR},${$E}[$ens]->{MINUTE},
- ${$E}[$ens]->{SECONDS},$B4)
+ ${$E}[$ens]->{SECOND},$B4)
= unpack('VvVVCCCCCCCCC',$buf);
${$E}[$ens]->{PRESSURE} /= 1000;
${$E}[$ens]->{PRESSURE_STDDEV} /= 1000;
${$E}[$ens]->{YEAR} *= 100; ${$E}[$ens]->{YEAR} += $B3;
- ${$E}[$ens]->{SECONDS} += $B4/100;
+ ${$E}[$ens]->{SECOND} += $B4/100;
}
# THE FOLLOWING LINE OF CODE WAS REMOVED 7/30/2016 WHEN I ADDED A POP
@@ -899,10 +917,10 @@
${$E}[$ens]->{TIME}
= sprintf("%02d:%02d:%05.02f",${$E}[$ens]->{HOUR},
${$E}[$ens]->{MINUTE},
- ${$E}[$ens]->{SECONDS});
+ ${$E}[$ens]->{SECOND});
${$E}[$ens]->{DAYNO}
= &_dayNo(${$E}[$ens]->{YEAR},${$E}[$ens]->{MONTH},${$E}[$ens]->{DAY},
- ${$E}[$ens]->{HOUR},${$E}[$ens]->{MINUTE},${$E}[$ens]->{SECONDS});
+ ${$E}[$ens]->{HOUR},${$E}[$ens]->{MINUTE},${$E}[$ens]->{SECOND});
# when analyzing an STA file from an OS75 SADCP (Poseidon),
# I noticed that there is no time information. This causes
@@ -918,7 +936,7 @@
${$E}[$ens]->{DAY},
${$E}[$ens]->{MONTH}-1, # timegm jan==0!!!
${$E}[$ens]->{YEAR})
- + ${$E}[$ens]->{SECONDS};
+ + ${$E}[$ens]->{SECOND};
$dayStart = timegm(0,0,0,${$E}[$ens]->{DAY},
${$E}[$ens]->{MONTH}-1,
@@ -1090,6 +1108,7 @@
sysseek(WBRF,$start_ens+$ens_length+2,0) || die("$WBRcfn: $!");
} # ens loop
}
+print(STDERR "\n") if ($global::readDataProgress > 0);
sub WBRdtaIndex($)
{
@@ -1121,8 +1140,10 @@
{
my($fn,$dta) = @_;
- die("writeData() needs \$WBRcfn from previous readData()")
+ die("writeData() needs \$WBRcfn from previous readData()\n")
unless (length($WBRcfn) > 0);
+ die("writeData() only works with clean PD0 files\n")
+ if ($RDI_PD0_IO::File_Dirty);
sysseek(WBRF,0,0) || die("$WBRcfn: $!"); # rewind input file
$WBPcfn = $fn; # set patch file name for error messages
@@ -1244,9 +1265,9 @@
$dta->{ENSEMBLE}[$ens]->{DAY},
$dta->{ENSEMBLE}[$ens]->{HOUR},
$dta->{ENSEMBLE}[$ens]->{MINUTE},
- $dta->{ENSEMBLE}[$ens]->{SECONDS},$B4) =
+ $dta->{ENSEMBLE}[$ens]->{SECOND},$B4) =
unpack('CCCCCCC',$buf);
- $dta->{ENSEMBLE}[$ens]->{SECONDS} += $B4/100;
+ $dta->{ENSEMBLE}[$ens]->{SECOND} += $B4/100;
$dta->{ENSEMBLE}[$ens]->{YEAR} += ($dta->{ENSEMBLE}[$ens]->{YEAR} > 80) ? 1900 : 2000;
}
@@ -1299,8 +1320,8 @@
my($century) = int($dta->{ENSEMBLE}[$ens]->{YEAR} / 100);
my($year) = $dta->{ENSEMBLE}[$ens]->{YEAR} % 100;
- my($seconds) = int($dta->{ENSEMBLE}[$ens]->{SECONDS});
- my($hundredths) = 100 * ($dta->{ENSEMBLE}[$ens]->{SECONDS} - $seconds);
+ my($seconds) = int($dta->{ENSEMBLE}[$ens]->{SECOND});
+ my($hundredths) = 100 * ($dta->{ENSEMBLE}[$ens]->{SECOND} - $seconds);
$buf = pack('CCCCCCCC',$century,$year,$dta->{ENSEMBLE}[$ens]->{MONTH},
$dta->{ENSEMBLE}[$ens]->{DAY},$dta->{ENSEMBLE}[$ens]->{HOUR},
$dta->{ENSEMBLE}[$ens]->{MINUTE},$seconds,$hundredths);
--- a/listBins
+++ b/listBins
@@ -2,9 +2,9 @@
#======================================================================
# L I S T B I N S
# doc: Fri Aug 25 15:57:05 2006
-# dlm: Thu Jun 13 22:17:02 2019
+# dlm: Thu Feb 13 10:36:24 2020
# (c) 2006 A.M. Thurnherr
-# uE-Info: 75 58 NIL 0 0 72 0 2 4 NIL ofnI
+# uE-Info: 133 33 NIL 0 0 72 0 2 4 NIL ofnI
#======================================================================
# Split data file into per-bin time series.
@@ -73,6 +73,7 @@
# Aug 29, 2018: - added error message on -r decoding failures
# Jun 13, 2018: - adpated to RTI files (disabled BIT error check)
# - BUG: dn did not have sufficient digits
+# Feb 13, 2020: - added -z
# General Notes:
# - everything (e.g. beams) is numbered from 1
@@ -117,7 +118,7 @@
require "$ANTS/ants.pl";
require "$ANTS/libconv.pl";
-die("Usage: $0 [-r)ange <first_ens,last_ens>] [-l)ast <bin>] [-R)enumber ensembles from 1] " .
+die("Usage: $0 [-z) progress dots] [-r)ange <first_ens,last_ens>] [-l)ast <bin>] [-R)enumber ensembles from 1] " .
"[-o)utput <redirection[>bin%d.raw]>] " .
"[output -a)ll ens (not just those with good vels)] " .
"[-M)agnetic <declination>] " .
@@ -127,7 +128,9 @@
"[require -4)-beam solutions] [-d)iscard <beam#>] " .
"[-p)ct-good <min>] " .
"<RDI file>\n")
- unless (&getopts("4aB:d:l:M:o:p:r:P:RS:T:") && @ARGV == 1);
+ unless (&getopts("4aB:d:l:M:o:p:r:P:RS:T:z") && @ARGV == 1);
+
+$global::readDataProgress = 10000 if defined($opt_z);
($P{pitch_bias},$P{roll_bias}) = split('[,/]',$opt_P);
($P{velbias_b1},$P{velbias_b2},$P{velbias_b3},$P{velbias_b4}) = split('[,/]',$opt_B);
--- a/listEns
+++ b/listEns
@@ -2,9 +2,9 @@
#======================================================================
# L I S T E N S
# doc: Sat Jan 18 18:41:49 2003
-# dlm: Thu May 31 11:10:51 2018
+# dlm: Thu Feb 13 10:36:50 2020
# (c) 2003 A.M. Thurnherr
-# uE-Info: 62 51 NIL 0 0 72 2 2 4 NIL ofnI
+# uE-Info: 96 54 NIL 0 0 72 2 2 4 NIL ofnI
#======================================================================
$synopsis = 'list ensemble summaries (default), dump ensembles (-E), time-average ensembles (-T)';
@@ -60,6 +60,7 @@
# - removed support for multiple input files
# Apr 10, 2018: - added -l)ast bin
# May 31, 2018: - BUG: -A was disabled by default
+# Feb 13, 2020: - added support for $readDataProgress
# Notes:
# - -E/-B outputs data in earth coordinates, unless -b is set also
@@ -92,15 +93,18 @@
"[require min -p)ercent-good <#>]\n\t\t" .
"[keep -b)eam coords (do not transform to earth coordinates)]\n\t" .
"Common Options:\n\t\t" .
+ "[-z print progress dots every 10000 ens\n\t\t" .
"[add -f)ields <[name=]FIELD[,...]>]\n\t\t" .
"[require -4)-beam solutions] [-d)iscard <beam#>]\n\t\t" .
"[-r)ange <first_ens,last_ens>] [-l)ast <bin>]\n\t\t" .
"[in-w)ater ensembles only]\n")
- unless (&getopts("4AB:bd:E:f:i:l:M:p:r:S:T:w") && @ARGV == 1);
+ unless (&getopts('4AB:bd:E:f:i:l:M:p:r:S:T:wz') && @ARGV == 1);
die("$ARGV[0]: no such file\n")
unless (-f $ARGV[0]);
+$global::readDataProgress = 10000 if defined($opt_z);
+
$dump_ens = defined($opt_E) + defined($opt_T);
die("$self: cannot combine -E with -T\n") if ($dump_ens > 1);
--- a/patchPD0
+++ b/patchPD0
@@ -2,9 +2,9 @@
#======================================================================
# P A T C H P D 0
# doc: Tue Aug 23 20:00:15 2016
-# dlm: Wed Jun 13 20:35:05 2018
+# dlm: Mon Jul 1 16:56:51 2019
# (c) 2010 A.M. Thurnherr
-# uE-Info: 184 88 NIL 0 0 72 2 2 4 NIL ofnI
+# uE-Info: 189 0 NIL 0 0 72 2 2 4 NIL ofnI
#======================================================================
$antsSummary = 'patch TRDI PD0 file with external attitude data';
@@ -19,6 +19,7 @@
# - BUG: not backward compatible with old IMP files any more
# Jun 13, 2017: - added pitch and roll to -o
# - BUG: ??? does -o handle pitch and roll ANOMALIES correctly?
+# Jun 30, 2019: - -o did not work with single argument
# PATCH-FILE REQUIREMENTS (ANTS format)
# - %LADCP_pitch.mu %LADCP_roll.mu mean LADCP pitch and roll
@@ -85,8 +86,6 @@
# Step 2: Process External Attidue Input to Patch PD0 file
#----------------------------------------------------------------------
-my($pr_missing,$hdg_missing) = (0,0);
-
&antsIn(); # load first IMP record
my($ensF) = &fnr('LADCP_ens');
@@ -102,9 +101,9 @@
if (defined($opt_o)) {
my($pofs,$rofs,$hofs) = split(/,/,$opt_o);
- if (defined($pofs)) { # pitch and roll offsets supplied
+ if (defined($rofs)) { # pitch and roll offsets supplied
croak("$0: cannot decode -o $opt_o\n")
- unless numbersp($pofs,$rofs);
+ unless numbersp($pofs,$rofs,$hofs);
} else { # no pitch and roll, only heading
$hofs = $pofs;
$pofs = undef;
@@ -128,13 +127,19 @@
}
}
+my($pr_missing,$hdg_missing) = (0,0);
+my($missing_pr_block_len,$missing_hdg_block_len);
+
do {
+ my($ADCP_deployed);
my($ens) = $P{RECNO};
die("assertion failed [$ants_[0][$ensF] != $LADCP{ENSEMBLE}[$ens]->{NUMBER} --- 1-$LADCP{ENSEMBLE}[0]->{NUMBER} + $P{RECNO} + $d]")
unless ($ants_[0][$ensF] == $LADCP{ENSEMBLE}[$ens]->{NUMBER});
$LADCP{ENSEMBLE}[$ens]->{DATA_SOURCE_ID} = 0xA0;
if (numbersp($ants_[0][$pitchF],$ants_[0][$rollF])) { # valid IMP data -> patch LADCP ensemble
+ $ADCP_deployed = 1;
+ $missing_pr_block_len = 0;
if (defined($opt_o)) { # -o set: rotate pitch and roll into correct coordinates
my($rot_p) = ($ants_[$r][$pitchF] * $crho +
$ants_[$r][$rollF] * $srho);
@@ -153,7 +158,8 @@
$LADCP{ENSEMBLE}[$ens]->{ROLL} = $LADCP_roll_mean + $ants_[0][$rollF];
}
} else { # no valid IMP pitch and roll => invalidate LADCP data
- $pr_missing++;
+ $pr_missing++ if $ADCP_deployed; # don't count missing before deployment
+ $missing_pr_block_len++;
unless ($opt_k) {
clearEns(\%LADCP,$ens);
$LADCP{ENSEMBLE}[$ens]->{DATA_SOURCE_ID}= 0xA0;
@@ -162,6 +168,7 @@
if (numberp($ants_[0][$hdgF])) { # valid IMP heading
$LADCP{ENSEMBLE}[$ens]->{DATA_SOURCE_ID} |= $opt_h;
+ $missing_hdg_block_len = 0;
if (defined($opt_o)) { # apply offset on -o; otherwise, data are correctly rotated
$ants_[0][$hdgF] -= $rho;
$ants_[0][$hdgF] += 360 if ($ants_[0][$hdgF] < 0);
@@ -169,13 +176,16 @@
$LADCP{ENSEMBLE}[$ens]->{HEADING} = $ants_[0][$hdgF] # patch heading
if $opt_h;
} else { # no valid IMP heading => invalidate LADCP data
- $hdg_missing++;
+ $hdg_missing++ if $ADCP_deployed;
+ $missing_hdg_block_len++;
unless ($opt_k) {
clearEns(\%LADCP,$ens);
$LADCP{ENSEMBLE}[$ens]->{DATA_SOURCE_ID}= 0xA0;
}
}
} while (&antsIn());
+$pr_missing -= missing_pr_block_len; # don't count final block (post recovery)
+$hdg_missing -= missing_hdg_block_len;
$LADCP{ENSEMBLE}[0]->{DATA_SOURCE_ID} = 0x7F; # ensure correct DSID (1st ens: orig; 2nd ens: this prog)
$LADCP{ENSEMBLE}[1]->{DATA_SOURCE_ID} = 0xA0