# HG changeset patch # User A.M. Thurnherr # Date 1514063632 18000 # Node ID 80d039881d2ce1b2a5503e0a69134e1638edc9a8 # Parent d7ab920c1de63d773cce40fb2eafb0e0ba96cd73 V2.1 diff --git a/ADCP_tools_lib.pl b/ADCP_tools_lib.pl --- a/ADCP_tools_lib.pl +++ b/ADCP_tools_lib.pl @@ -1,9 +1,9 @@ #====================================================================== # A D C P _ T O O L S _ L I B . P L # doc: Tue Jan 5 10:45:47 2016 -# dlm: Tue Nov 28 10:58:22 2017 +# dlm: Thu Dec 7 10:43:28 2017 # (c) 2016 A.M. Thurnherr -# uE-Info: 14 51 NIL 0 0 72 0 2 4 NIL ofnI +# uE-Info: 17 25 NIL 0 0 72 0 2 4 NIL ofnI #====================================================================== # HISTORY: @@ -12,8 +12,9 @@ # Aug 23, 2016: - updated to V1.8 # Mar 12, 2017: - updated to V1.9 for LADCP_w 1.3 # Nov 28, 2017: - updated to V2.0 for LADCP_w 1.4 +# Dec 7, 2017: - updated to V2.1 for improvements to listHdr -$ADCP_tools_version = 2.0; +$ADCP_tools_version = 2.1; die(sprintf("$0: obsolete ADCP_tools V%.1f; V%.1f required\n", $ADCP_tools_version,$ADCP_tools_minVersion)) diff --git a/HISTORY b/HISTORY --- a/HISTORY +++ b/HISTORY @@ -1,9 +1,9 @@ ====================================================================== H I S T O R Y doc: Tue May 15 18:04:39 2012 - dlm: Wed Dec 6 12:42:00 2017 + dlm: Sat Dec 23 16:13:38 2017 (c) 2012 A.M. Thurnherr - uE-Info: 199 15 NIL 0 0 72 3 2 4 NIL ofnI + uE-Info: 214 15 NIL 0 0 72 3 2 4 NIL ofnI ====================================================================== -------------------------------------- @@ -183,7 +183,7 @@ #------------------------------------- # V2.0 (bad bug in velBeamtoBPEarth()) --------------------------------------- +#------------------------------------- Nov 20, 2017: - added patchPD0 to DT tools @@ -197,3 +197,18 @@ - updated [Manual.pdf] - added [Manual.pdf] [loadANTS.m] to distro - published + +#---------------------------------------------------- +# V2.1 (make listHdr -s not bomb on incomplete files) +#---------------------------------------------------- + +Dec 7, 2017: + - changes to [RDI_PD0_IO.pl] [listHdr] + - updated to V2.1 [ADCP_tools_lib.pl] + +Dec 23, 2017: + - added support for -c to [patchPD0] for DT KVH system + - added backward compatibility to [patchPD0] + - updated all tools to use MinVersion 2.1 + - updated [patchPD0] to use ANTSlib V7.0 + - PUBLISHED diff --git a/RDI_PD0_IO.pl b/RDI_PD0_IO.pl --- a/RDI_PD0_IO.pl +++ b/RDI_PD0_IO.pl @@ -1,9 +1,9 @@ #====================================================================== # R D I _ P D 0 _ I O . P L # doc: Sat Jan 18 14:54:43 2003 -# dlm: Wed Nov 22 11:03:59 2017 +# dlm: Sat Dec 23 16:03:46 2017 # (c) 2003 A.M. Thurnherr -# uE-Info: 156 40 NIL 0 0 72 0 2 4 NIL ofnI +# uE-Info: 1184 24 NIL 0 0 72 2 2 4 NIL ofnI #====================================================================== # Read RDI PD0 binary data files (*.[0-9][0-9][0-9]) @@ -92,6 +92,11 @@ # Aug 8, 2017: - replaced croak() by die() # - added actual transducer frequencies # Nov 22, 2017: - BUG: dayNo() and monthLength() clashed with [libconv.pl] +# - added support for RDI_PD0_IO::IGNORE_Y2K_CLOCK +# Dec 7, 2017: - added suppress_error to readHeader() +# Dec 23, 2017: - BUG: could no longer read Anslope II raw files +# - added support for patching ADCP time data +# - added support for RDI_PD0_IO::OVERRIDE_Y2K_CLOCK # FIRMWARE VERSIONS: # It appears that different firmware versions generate different file @@ -391,10 +396,16 @@ sub readHeader(@) { - my($fn,$dta) = @_; + my($fn,$dta,$suppress_error) = @_; $WBRcfn = $fn; open(WBRF,$WBRcfn) || die("$WBRcfn: $!"); - WBRhdr($dta) || die("$WBRcfn: Insufficient data\n"); + if (WBRhdr($dta)) { + return 1; + } elsif ($suppress_error) { + return undef; + } else { + die("$WBRcfn: Insufficient data\n"); + } } sub WBRhdr(@) @@ -779,8 +790,8 @@ die(sprintf($FmtErr,$WBRcfn,"Variable Leader",$id,$ensNo + ($lastEns - ($lastEns & 0xFFFF)))); } -# if ($fixed_leader_bytes==42 || $fixed_leader_bytes==58) { # BB150 & Explorer DVL - sysread(WBRF,$buf,7) == 7 || die("$WBRcfn: $!"); +# if ($fixed_leader_bytes==42 || $fixed_leader_bytes==58) { # BB150 & Explorer DVL (if DISABLED!) + sysread(WBRF,$buf,7) == 7 || die("$WBRcfn: $!"); # always read pre-Y2K clock (${$E}[$ens]->{YEAR},${$E}[$ens]->{MONTH}, ${$E}[$ens]->{DAY},${$E}[$ens]->{HOUR},${$E}[$ens]->{MINUTE}, ${$E}[$ens]->{SECONDS},$B4) = unpack('CCCCCCC',$buf); @@ -852,7 +863,9 @@ # THE FOLLOWING LINE OF CODE WAS REMOVED 7/30/2016 WHEN I ADDED A POP # TO THE last STATEMENT ABOVE (INCOMPLETE ENSEMBLE) -# pop(@{$E}),last if (${$E}[$ens]->{MONTH}>12); # 10/15/2014; IWISE#145 UL ??? +# THE LINE WAS RE-ENABLED ON 12/23/2017 BECAUSE OTHERWISE THE +# ANSLOPE II PROFILES IN THE HOWTO CANNOT BE READ. + pop(@{$E}),last if (${$E}[$ens]->{MONTH}>12); # 10/15/2014; IWISE#145 UL ??? if ($fixed_leader_bytes == 58) { # Explorer DVL sysread(WBRF,$buf,14) == 14 || die("$WBRcfn: $!"); @@ -882,6 +895,7 @@ ${$E}[$ens]->{UNIX_TIME} = 0; ${$E}[$ens]->{SECNO} = 0; } else { +# print(STDERR "\n[$ens]->${$E}[$ens]->{MINUTE}:${$E}[$ens]->{HOUR},${$E}[$ens]->{DAY},${$E}[$ens]->{MONTH},${$E}[$ens]->{YEAR}-<\n"); ${$E}[$ens]->{UNIX_TIME} = timegm(0,${$E}[$ens]->{MINUTE}, ${$E}[$ens]->{HOUR}, @@ -1157,30 +1171,56 @@ sysseek(WBPF,$start_ens+$WBPofs[0]+25,0) || die("$WBPcfn: $!"); syswrite(WBPF,$buf,1) == 1 || die("$WBPcfn: $!"); - #------------------------------ - # Variable Leader - #------------------------------ + #---------------------------------------------------------------------- + # Variable Leader #1 + # - if $RDI_PD0_IO::OVERRIDE_Y2K_CLOCK is set, the data from the pre-Y2K + # clock are used to override the ADCP clock values; this allows + # a better time to be recorded by the data acquisition system + # without overwriting the main instrument clock data + #---------------------------------------------------------------------- - sysseek(WBPF,$start_ens+$WBPofs[1]+14,0) || die("$WBPcfn: $!"); # jump to SPEED_OF_SOUND + if ($RDI_PD0_IO::OVERRIDE_Y2K_CLOCK) { + sysseek(WBPF,$start_ens+$WBPofs[1]+4,0) || die("$WBPcfn: $!"); # jump to RTC_YEAR + sysread(WBPF,$buf,7) == 7 || die("$WBRcfn: $!"); # read pre-Y2K clock + ($dta->{ENSEMBLE}[$ens]->{YEAR}, + $dta->{ENSEMBLE}[$ens]->{MONTH}, + $dta->{ENSEMBLE}[$ens]->{DAY}, + $dta->{ENSEMBLE}[$ens]->{HOUR}, + $dta->{ENSEMBLE}[$ens]->{MINUTE}, + $dta->{ENSEMBLE}[$ens]->{SECONDS},$B4) = + unpack('CCCCCCC',$buf); + $dta->{ENSEMBLE}[$ens]->{SECONDS} += $B4/100; + $dta->{ENSEMBLE}[$ens]->{YEAR} += ($dta->{ENSEMBLE}[$ens]->{YEAR} > 80) ? 1900 : 2000; + } + + #---------------------------------------------------------------------- + # Variable Leader #2 + # - patch everything from SPEED_OF_SOUND to TEMPERATURE + # - at one stage, IMP allowed for missing values in pitch/roll and heading; + # on 12/23/2017 the corresponding code was disabled (replaced by assertion) + #---------------------------------------------------------------------- + + sysseek(WBPF,$start_ens+$WBPofs[1]+14,0) || die("$WBPcfn: $!"); # jump to SPEED_OF_SOUND $dta->{ENSEMBLE}[$ens]->{XDUCER_DEPTH} = _round($dta->{ENSEMBLE}[$ens]->{XDUCER_DEPTH}*10); - #--------------------------------- - # NB: IMP allows for missing value - #--------------------------------- +# $dta->{ENSEMBLE}[$ens]->{HEADING} = defined($dta->{ENSEMBLE}[$ens]->{HEADING}) +# ? _round($dta->{ENSEMBLE}[$ens]->{HEADING}*100) +# : 0xF000; +# $dta->{ENSEMBLE}[$ens]->{PITCH} = defined($dta->{ENSEMBLE}[$ens]->{PITCH}) +# ? unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{PITCH}*100))) +# : 0x8000; +# $dta->{ENSEMBLE}[$ens]->{ROLL} = defined($dta->{ENSEMBLE}[$ens]->{ROLL}) +# ? unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{ROLL}*100))) +# : 0x8000; - $dta->{ENSEMBLE}[$ens]->{HEADING} = defined($dta->{ENSEMBLE}[$ens]->{HEADING}) - ? _round($dta->{ENSEMBLE}[$ens]->{HEADING}*100) - : 0xF000; - $dta->{ENSEMBLE}[$ens]->{PITCH} = defined($dta->{ENSEMBLE}[$ens]->{PITCH}) - ? unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{PITCH}*100))) - : 0x8000; - $dta->{ENSEMBLE}[$ens]->{ROLL} = defined($dta->{ENSEMBLE}[$ens]->{ROLL}) - ? unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{ROLL}*100))) - : 0x8000; - - $dta->{ENSEMBLE}[$ens]->{TEMPERATURE} = - unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{TEMPERATURE}*100))); + croak("$0: assertion failed") unless defined($dta->{ENSEMBLE}[$ens]->{HEADING}) && + defined($dta->{ENSEMBLE}[$ens]->{PITCH}) && + defined($dta->{ENSEMBLE}[$ens]->{ROLL}); + $dta->{ENSEMBLE}[$ens]->{HEADING} = _round($dta->{ENSEMBLE}[$ens]->{HEADING}*100); + $dta->{ENSEMBLE}[$ens]->{PITCH} = unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{PITCH}*100))); + $dta->{ENSEMBLE}[$ens]->{ROLL} = unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{ROLL}*100))); + $dta->{ENSEMBLE}[$ens]->{TEMPERATURE} = unpack('S',pack('s',_round($dta->{ENSEMBLE}[$ens]->{TEMPERATURE}*100))); $buf = pack('vvvvvvv', $dta->{ENSEMBLE}[$ens]->{SPEED_OF_SOUND}, @@ -1192,6 +1232,23 @@ $nw == 14 || die("$WBPcfn: $nw bytes written ($!)"); + #---------------------------------------------------------------------- + # Variable Leader #3 + # - patch Y2K RTC + #---------------------------------------------------------------------- + + sysseek(WBPF,$start_ens+$WBPofs[1]+57,0) || die("$WBPcfn: $!"); # jump to RTC_CENTURY + + my($century) = int($dta->{ENSEMBLE}[$ens]->{YEAR} / 100); + my($year) = $dta->{ENSEMBLE}[$ens]->{YEAR} % 100; + my($seconds) = int($dta->{ENSEMBLE}[$ens]->{SECONDS}); + my($hundredths) = 100 * ($dta->{ENSEMBLE}[$ens]->{SECONDS} - $seconds); + $buf = pack('CCCCCCCC',$century,$year,$dta->{ENSEMBLE}[$ens]->{MONTH}, + $dta->{ENSEMBLE}[$ens]->{DAY},$dta->{ENSEMBLE}[$ens]->{HOUR}, + $dta->{ENSEMBLE}[$ens]->{MINUTE},$seconds,$hundredths); + my($nw) = syswrite(WBPF,$buf,8); + $nw == 8 || die("$WBPcfn: $nw bytes written ($!)"); + #-------------------- # Velocity Data #-------------------- diff --git a/editPD0 b/editPD0 --- a/editPD0 +++ b/editPD0 @@ -2,9 +2,9 @@ #====================================================================== # E D I T P D 0 # doc: Mon Nov 25 20:24:31 2013 -# dlm: Wed Dec 6 09:38:33 2017 +# dlm: Sat Dec 23 16:10:02 2017 # (c) 2013 A.M. Thurnherr -# uE-Info: 89 37 NIL 0 0 72 2 2 4 NIL ofnI +# uE-Info: 83 28 NIL 0 0 72 2 2 4 NIL ofnI #====================================================================== # edit RDI PD0 file, e.g. to replace pitch/roll/heading with external values @@ -80,7 +80,7 @@ use Getopt::Std; ($ADCP_TOOLS) = ($0 =~ m{(.*/)[^/]+}); -$ADCP_tools_minVersion = 1.4; +$ADCP_tools_minVersion = 2.1; require "$ADCP_TOOLS/ADCP_tools_lib.pl"; $USAGE = "$0 @ARGV"; diff --git a/listBins b/listBins --- a/listBins +++ b/listBins @@ -2,9 +2,9 @@ #====================================================================== # L I S T B I N S # doc: Fri Aug 25 15:57:05 2006 -# dlm: Mon Jun 19 03:49:12 2017 +# dlm: Sat Dec 23 16:10:11 2017 # (c) 2006 A.M. Thurnherr -# uE-Info: 168 1 NIL 0 0 72 0 2 4 NIL ofnI +# uE-Info: 104 28 NIL 0 0 72 0 2 4 NIL ofnI #====================================================================== # Split data file into per-bin time series. @@ -101,7 +101,7 @@ use Getopt::Std; -$ADCP_tools_minVersion = 1.4; +$ADCP_tools_minVersion = 2.1; ($ADCP_TOOLS) = ($0 =~ m{(.*/)[^/]+}); require "$ADCP_TOOLS/ADCP_tools_lib.pl"; diff --git a/listHdr b/listHdr --- a/listHdr +++ b/listHdr @@ -2,9 +2,9 @@ #====================================================================== # L I S T H D R # doc: Sat Jan 18 18:41:49 2003 -# dlm: Tue Aug 8 16:25:46 2017 +# dlm: Thu Dec 7 10:45:31 2017 # (c) 2003 A.M. Thurnherr -# uE-Info: 102 39 NIL 0 0 72 10 2 4 NIL ofnI +# uE-Info: 79 60 NIL 0 0 72 10 2 4 NIL ofnI #====================================================================== # Print useful info from the RDI BB header @@ -47,7 +47,7 @@ use Getopt::Std; ($ADCP_TOOLS) = ($0 =~ m{(.*/)[^/]+}); -$ADCP_tools_minVersion = 1.4; +$ADCP_tools_minVersion = 2.1; require "$ADCP_TOOLS/ADCP_tools_lib.pl"; die("Usage: $0 [-s)ummary] [-l)ong listing] \n") @@ -63,17 +63,21 @@ die("$ARGV[0]: No such file or directory\n") unless (-f $ARGV[0]); - readHeader($ARGV[0],\%hdr); + my($valid) = readHeader($ARGV[0],\%hdr,$opt_s); print("$ARGV[0]:\n") unless ($opt_s); if ($opt_s) { # summary ANTS output my($id) = $ARGV[0]; $id =~ s/00[0-9]\.000//; # leave just deployment name for std RDI files $id =~ s@^.*/([^/]+)@\1@; - printf("%s %d %.1f %d %g %d %.1f\n", - $id,$hdr{SERIAL_NUMBER},$hdr{BEAM_FREQUENCY}, - $hdr{N_BINS},$hdr{BLANKING_DISTANCE},$hdr{BIN_LENGTH}, - $hdr{TRANSMITTED_PULSE_LENGTH}); + if ($valid) { + printf("%s %d %.1f %d %g %d %.1f\n", + $id,$hdr{SERIAL_NUMBER},$hdr{BEAM_FREQUENCY}, + $hdr{N_BINS},$hdr{BLANKING_DISTANCE},$hdr{BIN_LENGTH}, + $hdr{TRANSMITTED_PULSE_LENGTH}); + } else { + printf("%s [no complete ensemble in file]\n",$id); + } shift; next; } diff --git a/listVels b/listVels --- a/listVels +++ b/listVels @@ -2,9 +2,9 @@ #====================================================================== # L I S T V E L S # doc: Mon Apr 25 21:12:54 2016 -# dlm: Mon Apr 25 21:48:18 2016 +# dlm: Sat Dec 23 16:10:33 2017 # (c) 2016 A.M. Thurnherr -# uE-Info: 180 0 NIL 0 0 72 10 2 4 NIL ofnI +# uE-Info: 21 28 NIL 0 0 72 10 2 4 NIL ofnI #====================================================================== # list water-track velocity samples as ANTS records (PD02ANTS) @@ -18,7 +18,7 @@ use Getopt::Std; -$ADCP_tools_minVersion = 1.4; +$ADCP_tools_minVersion = 2.1; ($ADCP_TOOLS) = ($0 =~ m{(.*/)[^/]+}); require "$ADCP_TOOLS/ADCP_tools_lib.pl"; diff --git a/patchPD0 b/patchPD0 --- a/patchPD0 +++ b/patchPD0 @@ -2,9 +2,9 @@ #====================================================================== # P A T C H P D 0 # doc: Tue Aug 23 20:00:15 2016 -# dlm: Mon Nov 20 12:27:15 2017 +# dlm: Sat Dec 23 16:12:01 2017 # (c) 2010 A.M. Thurnherr -# uE-Info: 32 16 NIL 0 0 72 0 2 4 NIL ofnI +# uE-Info: 51 24 NIL 0 0 72 0 2 4 NIL ofnI #====================================================================== $antsSummary = 'patch TRDI PD0 file with external attitude data'; @@ -14,6 +14,9 @@ # Aug 25, 2016: - completed basic structure # Nov 20, 2017: - major code cleanup # - added -d) to keep original data source id +# Dec 9, 2017: - added $antsSuppressCommonOptions = 1; +# Dec 23, 2017: - added support for -c +# - BUG: not backward compatible with old IMP files any more # PATCH-FILE REQUIREMENTS (ANTS format) # - %LADCP_pitch.mu %LADCP_roll.mu mean LADCP pitch and roll @@ -45,8 +48,8 @@ ($ANTS) = (`which ANTSlib` =~ m{^(.*)/[^/]*$}); ($ADCP_TOOLS) = ($0 =~ m{^(.*)/[^/]*$}); -$antsMinLibVersion = 6.7; -$ADCP_tools_minVersion = 1.8; +$antsMinLibVersion = 7.0; +$ADCP_tools_minVersion = 2.1; require "$ANTS/ants.pl"; require "$ANTS/libvec.pl"; @@ -54,8 +57,10 @@ require "$ADCP_TOOLS/ADCP_tools_lib.pl"; $antsParseHeader = 0; -&antsUsage('dhko:pr',2, +$antsSuppressCommonOptions = 1; +&antsUsage('cdhko:pr',2, '[patch -p)itch] [-r)oll] [-h)eading] (none patches all)', + '[patch -c)lock with pre-Y2K RTC calues]', '[-o) ] [-k)eep velocities of unpatched ensembles]', '[keep original -d)ata-source id]', ' [external attitude file]'); @@ -63,6 +68,8 @@ $opt_p = $opt_r = $opt_h = 1 unless ($opt_p || $opt_r || $opt_h); +$RDI_PD0_IO::OVERRIDE_Y2K_CLOCK = $opt_c; + $LADCP_file = &antsFileArg(); $outPD0 = $ARGV[0]; shift; @@ -89,6 +96,8 @@ my($rho,$crho,$srho); if (defined($opt_o)) { + &antsAddParams('IMU_hdg_offset',$P{IMP_hdg_offset}) # backward compatibility + if defined($P{IMP_hdg_offset}); $rho = $opt_o - &antsRequireParam('IMU_hdg_offset'); $crho = cos(rad($rho)); $srho = sin(rad($rho)); @@ -98,7 +107,6 @@ my($ens) = $P{RECNO}; die("assertion failed [$ants_[0][$ensF] != $LADCP{ENSEMBLE}[$ens]->{NUMBER} --- 1-$LADCP{ENSEMBLE}[0]->{NUMBER} + $P{RECNO} + $d]") unless ($ants_[0][$ensF] == $LADCP{ENSEMBLE}[$ens]->{NUMBER}); -# print(STDERR "assertion succeeded [$ants_[0][$ensF] == $LADCP{ENSEMBLE}[$ens]->{NUMBER} --- 1-$LADCP{ENSEMBLE}[0]->{NUMBER} + $P{RECNO} + $d]\n"); $LADCP{ENSEMBLE}[$ens]->{DATA_SOURCE_ID} = 0xA0; if (numbersp($ants_[0][$pitchF],$ants_[0][$rollF])) {