#======================================================================

#                    R D I _ P D 0 _ I O . P L 

#                    doc: Sat Jan 18 14:54:43 2003

#                    (c) 2003 A.M. Thurnherr

#======================================================================

# Read RDI PD0 binary data files (*.[0-9][0-9][0-9])

# HISTORY:

#	Jan 18, 2003: - incepted aboard the Aurora Australis (KAOS)

#	Jan 19, 2003: - continued

#	Jan 20, 2003: - replaced INTENSITY by AMPLITUDE

#	Jan 21, 2003: - changed heading-correction field names

#	Jan 27, 2003: - cosmetics

#	Feb 14, 2003: - moved BT setup into header

#	Mar 15, 2003: - moved 10th xmit voltage into header as BATTERY field

#				  - removed again, because values are not meaningful

#	Feb 24, 2004: - continued aboard Nathaniel B. Palmer (Anslope II)

#				  - BUG: ensemble # was wrong on error messages

#	Feb 26, 2004: - removed ESW_ERROR and bitmasking of ERROR_STATUS_WORD

#	Feb 27, 2004: - removed some unused (commented-out) baggage

#	Mar 11, 2004: - BUG: renamed ACD -> ADC

#	Mar 18, 2004: - cosmetics

#	Mar 30, 2004: - rewrote to speed up reading; new version takes

#					~40% less time

#	Sep 14, 2005: - made BT data optional (NUMBER_OF_DATA_TYPES)

#				  - added DATA_FORMAT

#	Sep 15, 2005: - debugged

#				  - implement checksum to robustly find EOF

#				  - renamed to RDI_BB_Read.pl

#				  - BUG: had used POSIX::mktime with wrong month def!

#	Oct 30, 2005: - added WH300 FW16.27 file format

#				  - added DATA_FORMAT_VARIANT

#				  - changed semantics so that first valid ensemble is

#					in E[0] (instead of E[$ensNo-1])

#	Nov  8, 2005: - replaced UNIXTIME by UNIX_TIME

#				  - added SECNO

#	Aug 31: 2006: - added DAYNO

#	Aug  1, 2007: - BUG: typo in monthLength()

#	Sep 18, 2007: - modified readHeader() readDta() WBRhdr() WBRens() to

#					conserve memory (no large arrays as retvals)

#	Jun  4, 2008: - BUG: BB150 code was not considered on Sep 18, 2007

#	Aug 15, 2010: - downgraded "unexpected number of data types" from error to warning

#				  - BUG: WBRcfn had not been set correctly

#				  - modified to allow processing files without time info

#	May 12, 2011: - added code to report built-in-test errors

#	Mar 19, 2013: - added support for WH600 data file (58 fixed leader bytes)

#	Mar 20, 2013: - removed DATA_FORMAT stuff

#				  - added support for BT data in subset of ensembles

#	Apr 29, 2013: - changed semantics to assume EOF when unexpected number of data types

#					are present in an ensemble

#	Nov 25, 2013: - renamed from [RDI_BB_Read.pl]

#				  - begin implementing WBWens()

#				  - checkEnsemble() expunged

#	Mar  3, 2014: - BUG: WBPens() did not handle incomple ensembles at EOF correctly

#	Mar  4, 2014: - added support for DATA_SOURCE_ID

#	Apr 24, 2014: - added debug statements to log %-GOOD values

#	May  6, 2014: - loosened input format checks

#	May  7, 2014: - removed BT_present flag

#	Sep  6, 2014: - adapted WBRhdr to >7 data types

#	Oct 15, 2014: - implemented work-around for readData() not recognizing

#					incomplete ensemble at the end, which seems to imply that there is

#					a garbage final ensemble that passes the checksum test???

#	Oct  2, 2015: - added &skip_initial_trash()

#	Dec 18, 2015: - added most data types to WBPofs()

#				  - BUG: WBPens() requires round() for scaled values

#	Jan  9, 2016: - removed system() from writeData()

#				  - BUG: WBRhdr() did not set DATA_SOURCE_ID

#				  - added PRODUCER

#				  - BUG: writeData() did not work correctly for ECOGIG OC26 moored data (spaces in filename?)

#				  - added support for patching coordinate system

#	Feb 16, 2016: - added transducer orientation to WBPens()

#				  - BUG: most WBPens() error messages used wrong file name

#	Feb 23, 2016: - changed WBRhdr() to use 2nd ensemble (with correct data-source id)

#	Feb 26, 2016: - added basic BT data to WBPens(); not BT_RL_* and BT_SIGNAL_STRENGTH

#	Feb 29, 2016: - LEAP DAY: actually got BT data patching to work

#	Jul 30, 2016: - BUG: incomplete last ensemble with garbage content was returned on reading

#						 WH300 data

#	Aug  5, 2016: - cosmetics

#	Aug 23, 2016: - added &clearEns()

#	Nov  9, 2016: - made WBRhdr() return undef on "empty" files

#	Nov 18, 2016: - BUG: ensNo was not reported correctly in format errors

#	Nov 23, 2016: - no longer set pitch/roll/heading to undef in clearEns()

#	Mar  7, 2016: - renamed round() to stop clashing with ANTSLIB

#	May 18, 2017: - maded readHeader() more permissive (checkFmt flag)

#				  - added partial support for Ocean Surveyor data

#	Aug  1, 2017: BUG: minor typo in ocean surveyor code (err check not done?)

#	Aug  7, 2017: - added LAG_LENGTH

#				  - added SPEED_OF_SOUND to header

#	Aug  8, 2017: - replaced croak() by die()

#				  - added actual transducer frequencies

#	Nov 22, 2017: - BUG: dayNo() and monthLength() clashed with [libconv.pl]

#				  - added support for RDI_PD0_IO::IGNORE_Y2K_CLOCK

#	Dec  7, 2017: - added suppress_error to readHeader()

#	Dec 23, 2017: - BUG: could no longer read Anslope II raw files

#				  - added support for patching ADCP time data

#				  - added support for RDI_PD0_IO::OVERRIDE_Y2K_CLOCK

# FIRMWARE VERSIONS:

#	It appears that different firmware versions generate different file

#	structures. Currently (Sep 2005) these routines have been tested

#	with the following firmware versions (as reported by [listHdr]):

#

#	Firmw.	DATA_FORMAT(_VARIANT)	Owner	Cruise	FIXED_LEADER_LENGTH

#------------------------------------------------------------

#	05.52	BB150 (1)				UH		CLIVAR/P16S 42

#	16.12	WH300 (1)				FSU 	A0304		53

#	16.21	WH300 (1)				LDEO	NBP0402 	53

#	16.27	WH300 (2)				Nash	?			59

# PD0 FILE FORMAT EXTENSIONS:

#

#	- file creator encoded in DATA_SOURCE_ID

#

#	- first ensemble uses default RDI DATA_SOURCE_ID because the LDEO_IX

#	  software assumes this

#

#	- DATA_SOURCE_ID = 0x7F 					original TRDI PD0 file

#

#		PATCHED_MASK & 0x04:						pitch value has been patched

#		PATCHED_MASK & 0x02:						roll value has been patched

#		PATCHED_MASK & 0x01:						heading value has been patched

#			- PITCH & ROLL can be missing (0x8000 badval as in velocities)

#			- HEADING can be missing (0xF000 badval, as 0x8000 is valid 327.68 heading)

#

#	- DATA_SOURCE_ID = 0xE0 					produced by editPD0

# NOTES:

#	- RDI stores data in VAX/Intel byte order (little endian)

#	- the output data structure does not exactly mirror the file data

#	  structure; the header is not stored at all and the fixed leader

#	  data are not duplicated in every ensemble

#	- in the RDI files some fields that logically belong into the header

#	  or the fixed leader (e.g. BT_MIN_CORRELATION) appear in the 

#	  ensemble data --- these are not read on input

#	- the field names are generally unabbreviated except for

#	  BT (= Bottom Track), RL (= Reference Layer), MIN and MAX

#	- all arrays are 0-referenced, but the ensemble number is not!

#	- a list of filenames can be be passed to readData() so that

#	  files split onto several memory cards (typically .000 .001 &c)

#	  can be read --- not sure if this works, actually

#	- the RDI manuals are not entirely clear everywhere; I have made

#	  guesses in some cases, but they should not affect the main

#	  fields of interest

#	- some fields in the fixed leader are not really fixed in a LADCP

#	  setting (e.g. xducer orientation); I'v made an educated guess

#	  as to which fields to move to the ENS array

#	- all units except pressure are SI, i.e. in m and m/s

#	- I don't understand the ERROR_STATUS_WORD; here's what 3 different

#	  instruments returned:

#		0x88000100	FSU instrument during A0304 (Firmware 16.12)

#		0x88008180	LDEO uplooker (slave) during NBP0402 (Firmware 16.21)

#		0x00008180	LDEO downlooker (master) during NBP0402 (Firmware 16.21)

#	  According to the manual (January 2001 version) this would, for example,

#	  indicate power failures on both FSU and LDEO slave instruments...

#	- defining the variable "$RDI_PD0_IO::IGNORE_Y2K_CLOCK" before calling &readData()

#	  makes the code ignore the Y2K clock and use the old clock instead; this 

#	  is used for Dan Torres' KVH system

# &readData() returns perl obj (ref to anonymous hash) with the following

# structure:

#

#	DATA_SOURCE_ID					scalar		0x7f (Workhorse, also DVL)

#	NUMBER_OF_DATA_TYPES			scalar		6 (no BT) or 7

#	ENSEMBLE_BYTES					scalar		?, number of bytes w/o checksum

#	HEADER_BYTES					scalar		?

#	FIXED_LEADER_BYTES				scalar		42 for BB150; 53 for WH300, 58 for WH600, 59 for WH300(Nash)

#	VARIABLE_LEADER_BYTES			scalar		?

#	VELOCITY_DATA_BYTES 			scalar		?

#	CORRELATION_DATA_BYTES			scalar		?

#	ECHO_INTENSITY_DATA_BYTES		scalar		?

#	PERCENT_GOOD_DATA_BYTES 		scalar		?

#	BT_PRESENT						bool		NUMBER_OF_DATA_TYPES == 7

#	BT_DATA_BYTES					scalar		undefined, ? if BT_PRESENT

#	CPU_FW_VER						scalar		0--255

#	CPU_FW_REV						scalar		0--255

#	BEAM_FREQUENCY					scalar		75, 150, 300, 600, 1200, 2400 [kHz]

#	LAG_LENGTH						scalar		???

#	CONVEX_BEAM_PATTERN 			bool		undefined, 1

#	CONCAVE_BEAM_PATTERN			bool		undefined, 1

#	SENSOR_CONFIG					scalar		1--3

#	XDUCER_HEAD_ATTACHED			bool		undefined, 1

#	BEAM_ANGLE						scalar		15,20,30,undefined=other [deg]

#	N_BEAMS 						scalar		4--5

#	N_DEMODS						scalar		2--3(???),undefined=n/a

#	N_BINS							scalar		1--128

#	PINGS_PER_ENSEMBLE				scalar		0--16384

#	BIN_LENGTH						scalar		0.01--64 [m]

#	BLANKING_DISTANCE				scalar		0-99.99 [m]

#	MIN_CORRELATION 				scalar		0--255

#	N_CODE_REPETITIONS				scalar		0--255

#	MIN_PERCENT_GOOD				scalar		1--100 [%]

#	MAX_ERROR_VELOCITY				scalar		0--5 [m/s]

#	TIME_BETWEEN_PINGS				scalar		0--? [s]

#	BEAM_COORDINATES				bool		undefined,1

#	INSTRUMENT_COORDINATES			bool		undefined,1

#	SHIP_COORDINATES				bool		undefined,1

#	EARTH_COORDINATES				bool		undefined,1

#	PITCH_AND_ROLL_USED 			bool		undefined,1

#	USE_3_BEAM_ON_LOW_CORR			bool		undefined,1

#	BIN_MAPPING_ALLOWED 			bool		undefined,1

#	HEADING_ALIGNMENT				scalar		-179.99..180 [deg]

#	HEADING_BIAS					scalar		-179.99..180 [deg]

#	CALCULATE_SPEED_OF_SOUND		bool		undefined,1

#	USE_PRESSURE_SENSOR 			bool		undefined,1

#	USE_COMPASS 					bool		undefined,1

#	USE_PITCH_SENSOR				bool		undefined,1

#	USE_ROLL_SENSOR 				bool		undefined,1

#	USE_CONDUCTIVITY_SENSOR 		bool		undefined,1

#	USE_TEMPERATURE_SENSOR			bool		undefined,1

#	SPEED_OF_SOUND_CALCULATED		bool		undefined,1

#	PRESSURE_SENSOR_AVAILABLE		bool		undefined,1

#	COMPASS_AVAILABLE				bool		undefined,1

#	PITCH_SENSOR_AVAILABLE			bool		undefined,1

#	ROLL_SENSOR_AVAILABLE			bool		undefined,1

#	CONDUCTIVITY_SENSOR_AVAILABLE	bool		undefined,1

#	TEMPERATURE_SENSOR_AVAILABLE	bool		undefined,1

#	DISTANCE_TO_BIN1_CENTER 		scalar		0--655.35 [m]

#	TRANSMITTED_PULSE_LENGTH		scalar		0--655.35 [m]

#	RL_FIRST_BIN					scalar		1--128

#	RL_LAST_BIN 					scalar		1--128

#	FALSE_TARGET_THRESHOLD			scalar		0--254, undefined=disabled

#	LOW_LATENCY_SETTING 			scalar		0--5(???)

#	TRANSMIT_LAG_DISTANCE			scalar		0--655.35 [m]

#	CPU_SERIAL_NUMBER				scalar		undefined, 0--65535 if WH300

#	NARROW_BANDWIDTH				bool		undefined,1 (only set if WH300)

#	WIDE_BANDWIDTH					bool		undefined,1 (only set if WH300)

#	TRANSMIT_POWER					scalar		undefined, 0--255(high) if WH300

#	TRANSMIT_POWER_HIGH 			bool		undefined,1 (only set if WH300)

#	BT_PINGS_PER_ENSEMBLE			scalar		0--999

#	BT_DELAY_BEFORE_REACQUIRE		scalar		0--999

#	BT_MIN_CORRELATION				scalar		0--255

#	BT_MIN_EVAL_AMPLITUDE			scalar		0--255

#	BT_MIN_PERCENT_GOOD 			scalar		0--100 [%]

#	BT_MODE 						scalar		4,5,6(?)

#	BT_MAX_ERROR_VELOCITY			scalar		0--5 [m/s], undef=not screened

#	BT_RL_MIN_SIZE					scalar		0--99.9 [m]

#	BT_RL_NEAR						scalar		0--999.9 [m]

#	BT_RL_FAR						scalar		0--999.9 [m]

#	BT_MAX_TRACKING_DEPTH			scalar		8--999.9 [m]

#	ENSEMBLE[ensemble_no-1] 		array		ensemble info

#		XDUCER_FACING_UP			bool		undefined, 1

#		XDUCER_FACING_DOWN			bool		undefined, 1

#		N_BEAMS_USED				scalar		3,4,5(?)

#		NUMBER						scalar		1--16777215

#		BUILT_IN_TEST_ERROR 		scalar		?,undefined=none

#		SPEED_OF_SOUND				scalar		1400--1600 [m/s]

#		XDUCER_DEPTH				scalar		0.1--999.9 [m]

#		HEADING 					scalar		0--359.99 [deg]    --- IMP EXTENSION: undef

#		PITCH						scalar		-20.00-20.00 [deg] --- IMP EXTENSION: undef

#		ROLL						scalar		-20.00-20.00 [deg] --- IMP EXTENSION: undef

#		SALINITY					scalar		0-40 [psu]

#		TEMPERATURE 				scalar		-5.00--40.00 [deg]

#		MIN_PRE_PING_WAIT_TIME		scalar		? [s]

#		HEADING_STDDEV				scalar		0-180 [deg]

#		PITCH_STDDEV				scalar		0.0-20.0 [deg]

#		ROLL_STDDEV 				scalar		0.0-20.0 [deg]

#		ADC_XMIT_CURRENT			scalar		0--255

#		ADC_XMIT_VOLTAGE			scalar		0--255

#		ADC_AMBIENT_TEMPERATURE 	scalar		0--255

#		ADC_PRESSURE_PLUS			scalar		0--255

#		ADC_PRESSURE_MINUS			scalar		0--255

#		ADC_ATTITUDE_TEMPERATURE	scalar		0--255

#		ADC_ATTITUDE				scalar		0--255

#		ADC_CONTAMINATION			scalar		0--255

#		ERROR_STATUS_WORD			scalar		undefined, ? (only set if WH300)

#		PRESSURE					scalar		undefined, ?-? [dbar] (only set if WH300)

#		PRESSURE_STDDEV 			scalar		undefined, ?-? [dbar] (only set if WH300)

#		DATE						string		MM/DD/YYYY

#		YEAR						scalar		?

#		MONTH						scalar		1--12

#		DAY 						scalar		1--31

#		TIME						string		HH:MM:SS.hh

#		HOUR						scalar		0--23

#		MINUTE						scalar		0--59

#		SECONDS 					scalar		0--59.99

#		UNIX_TIME					scalar		0--?

#		SECNO						scalar		0--? (number of seconds since daystart)

#		DAYNO						double		fractional day number since start of current year (1.0 is midnight Jan 1st)

#		VELOCITY[bin][beam] 		scalars 	-32.767--32.768 [m/s], undef=bad

#		CORRELATION[bin][beam]		scalars 	1--255, undefined=bad

#		ECHO_AMPLITUDE[bin][beam]	scalars 	0--255

#		PERCENT_GOOD[bin][beam] 	scalars 	0--255

#		BT_RANGE[beam]				scalars 	tons [m]

#		BT_VELOCITY[beam]			scalars 	see VELOCITY

#		BT_CORRELATION[beam]		scalars 	see CORRELATION

#		BT_EVAL_AMPLITUDE[beam] 	scalars 	0--255

#		BT_PERCENT_GOOD[beam]		scalars 	see PERCENT_GOOD

#		BT_RL_VELOCITY[beam]		scalars 	see VELOCITY

#		BT_RL_CORRELATION[beam] 	scalars 	see CORRELATION

#		BT_RL_ECHO_AMPLITUDE[beam]	scalars 	see ECHO_AMPLITUDE

#		BT_RL_PERCENT_GOOD[beam]	scalars 	see PERCENT_GOOD

#		BT_SIGNAL_STRENGTH[beam]	scalars 	0--255

#		HIGH_GAIN					bool		1, undefined

#		LOW_GAIN					bool		1, undefined

	use strict;

	use Time::Local;						# timegm()

#----------------------------------------------------------------------

# Time Conversion Subroutines

#	- prepended with _ to avoid conflicts with [libconv.pl]

#----------------------------------------------------------------------

	sub _monthLength($$) 									# of days in month

	{

		my($Y,$M) = @_;

		return 31 if ($M==1 || $M==3 || $M==5 || $M==7 ||

					  $M==8 || $M==10 || $M==12);

		return 30 if ($M==4 || $M==6 || $M==9 || $M==11);

		return 28 if ($Y%4 != 0);

		return 29 if ($Y%100 != 0);

		return 28 if ($Y%400 > 0);

		return 29;

	}

	{ my($epoch,$lM,$lD,$lY,$ldn);							# static scope

	  sub _dayNo($$$$$$)

	  {

		  my($Y,$M,$D,$h,$m,$s) = @_;

		  my($dn);

		  if ($Y==$lY && $M==$lM && $D==$lD) {				# same day as last samp

			  $dn = $ldn;

		  } else {											# new day

			  $epoch = $Y unless defined($epoch);			# 1st call

			  $lY = $Y; $lM = $M; $lD = $D; 				# store

			  for ($dn=0,my($cY)=$epoch; $cY<$Y; $cY++) {	# multiple years

				  $dn += 337 + &_monthLength($Y,$M);

			  }

			  $dn += $D;									# day in month

			  while (--$M > 0) {							# preceding months

				  $dn += &_monthLength($Y,$M);

			  }

			  $ldn = $dn;									# store

		  }

		  return $dn + $h/24 + $m/24/60 + $s/24/3600;

	  }

	} # static scope

#----------------------------------------------------------------------

# Read Data

#----------------------------------------------------------------------

	my($WBRcfn,$WBPcfn);									# current file names for reading/patching

	my($BIT_errors) = 0;									# built-in-test errors

	my($FmtErr) = "%s: illegal %s Id 0x%04x at ensemble %d";

#----------------------------------------------------------------------

#----------------------------------------------------------------------

		}

	}

#----------------------------------------------------------------------

# readHeader(file_name,^dta) WBRhdr(^data)

#	- read header data

#	- also includes some data from 1st ens

#----------------------------------------------------------------------

sub readHeader(@)

{

	my($fn,$dta,$suppress_error) = @_;

	$WBRcfn = $fn;

	open(WBRF,$WBRcfn) || die("$WBRcfn: $!");

	if (WBRhdr($dta)) {

		return 1;

    } elsif ($suppress_error) {

		return undef;

    } else {

		die("$WBRcfn: Insufficient data\n");

    }

}

sub WBRhdr(@)

{

	my($dta,$checkFmt) = @_;

	my($start_ens,$buf,$hid,$did,$Ndt,$B,$W,$i,$dummy,$id,@WBRofs);

	my($B1,$B2,$B3,$B4,$B5,$B6,$B7,$W1,$W2,$W3,$W4,$W5);

	my($BT_dt);

	#--------------------

	# HEADER

	#--------------------

	sysread(WBRF,$buf,6) == 6 || return undef;

	($hid,$did,$dta->{ENSEMBLE_BYTES},$dummy,$dta->{NUMBER_OF_DATA_TYPES})

		= unpack('CCvCC',$buf);

	$hid == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (hid)",$hid,0));

	$did == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (did)",$did,0));

	$start_ens = sysseek(WBRF,$dta->{ENSEMBLE_BYTES}-6+2,1) || return undef;

	sysread(WBRF,$buf,6) == 6 || return undef;

	($hid,$did,$dta->{ENSEMBLE_BYTES},$dummy,$dta->{NUMBER_OF_DATA_TYPES})

		= unpack('CCvCC',$buf);

	$hid == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (hid2)",$hid,0));

	$dta->{DATA_SOURCE_ID} = $did;

	if ($did == 0x7f) {

		$dta->{PRODUCER} = 'TRDI ADCP';

	} elsif (($did&0xF0) == 0xA0) {

		$dta->{PRODUCER} = 'IMP+LADCP (Thurnherr software)';

	} elsif (($did&0xF0) == 0xE0) {

		$dta->{PRODUCER} = 'editPD0 (Thurnherr software)';

	} else {

		$dta->{PRODUCER} = sprintf('unknown (0x%02X)');

	}

	if ($checkFmt) {

		printf(STDERR "WARNING: unexpected number of data types (%d)\n",

			$dta->{NUMBER_OF_DATA_TYPES})

				unless ($dta->{NUMBER_OF_DATA_TYPES} == 6 ||

						$dta->{NUMBER_OF_DATA_TYPES} == 7);

		$dta->{BT_PRESENT} = ($dta->{NUMBER_OF_DATA_TYPES} >= 7);

	}

	sysread(WBRF,$buf,2*$dta->{NUMBER_OF_DATA_TYPES})

		== 2*$dta->{NUMBER_OF_DATA_TYPES}

			|| die("$WBRcfn: $!");

	@WBRofs = unpack("v$dta->{NUMBER_OF_DATA_TYPES}",$buf);

#	for ($i=0; $i<$dta->{NUMBER_OF_DATA_TYPES}; $i++) {

#	}

	$dta->{HEADER_BYTES}					= $WBRofs[0];

	$dta->{FIXED_LEADER_BYTES}				= $WBRofs[1] - $WBRofs[0];