#======================================================================

#                    doc: Sat Jan 18 14:54:43 2003

#                    (c) 2003 A.M. Thurnherr

#======================================================================

# Read RDI PD0 binary data files (*.[0-9][0-9][0-9])

# HISTORY:

#	Jan 18, 2003: - incepted aboard the Aurora Australis (KAOS)

#	Jan 19, 2003: - continued

#	Jan 20, 2003: - replaced INTENSITY by AMPLITUDE

#	Jan 21, 2003: - changed heading-correction field names

#	Jan 27, 2003: - cosmetics

#	Feb 14, 2003: - moved BT setup into header

#	Mar 15, 2003: - moved 10th xmit voltage into header as BATTERY field

#				  - removed again, because values are not meaningful

#	Feb 24, 2004: - continued aboard Nathaniel B. Palmer (Anslope II)

#				  - BUG: ensemble # was wrong on error messages

#	Feb 26, 2004: - removed ESW_ERROR and bitmasking of ERROR_STATUS_WORD

#	Feb 27, 2004: - removed some unused (commented-out) baggage

#	Mar 11, 2004: - BUG: renamed ACD -> ADC

#	Mar 18, 2004: - cosmetics

#	Mar 30, 2004: - rewrote to speed up reading; new version takes

#					~40% less time

#	Sep 14, 2005: - made BT data optional (NUMBER_OF_DATA_TYPES)

#				  - added DATA_FORMAT

#	Sep 15, 2005: - debugged

#				  - implement checksum to robustly find EOF

#				  - renamed to RDI_BB_Read.pl

#				  - BUG: had used POSIX::mktime with wrong month def!

#	Oct 30, 2005: - added WH300 FW16.27 file format

#				  - added DATA_FORMAT_VARIANT

#				  - changed semantics so that first valid ensemble is

#	Nov  8, 2005: - replaced UNIXTIME by UNIX_TIME

#				  - added SECNO

#	Aug 31: 2006: - added DAYNO

#	Aug  1, 2007: - BUG: typo in monthLength()

#	Sep 18, 2007: - modified readHeader() readDta() WBRhdr() WBRens() to

#					conserve memory (no large arrays as retvals)

#	Jun  4, 2008: - BUG: BB150 code was not considered on Sep 18, 2007

#	Aug 15, 2010: - downgraded "unexpected number of data types" from error to warning

#				  - BUG: WBRcfn had not been set correctly

#				  - modified to allow processing files without time info

#	May 12, 2011: - added code to report built-in-test errors

#	Mar 19, 2013: - added support for WH600 data file (58 fixed leader bytes)

#	Mar 20, 2013: - removed DATA_FORMAT stuff

#				  - added support for BT data in subset of ensembles

#	Apr 29, 2013: - changed semantics to assume EOF when unexpected number of data types

#					are present in an ensemble

#	Nov 25, 2013: - renamed from [RDI_BB_Read.pl]

#				  - begin implementing WBWens()

#				  - checkEnsemble() expunged

#	Mar  3, 2014: - BUG: WBPens() did not handle incomple ensembles at EOF correctly

#	Mar  4, 2014: - added support for DATA_SOURCE_ID

#	Apr 24, 2014: - added debug statements to log %-GOOD values

#	May  6, 2014: - loosened input format checks

#	May  7, 2014: - removed BT_present flag

#	Sep  6, 2014: - adapted WBRhdr to >7 data types

#	Oct 15, 2014: - implemented work-around for readData() not recognizing

#					incomplete ensemble at the end, which seems to imply that there is

#	Oct  2, 2015: - added &skip_initial_trash()

#	Dec 18, 2015: - added most data types to WBPofs()

#				  - BUG: WBPens() requires round() for scaled values

#	Jan  9, 2016: - removed system() from writeData()

#				  - BUG: WBRhdr() did not set DATA_SOURCE_ID

#				  - added PRODUCER

#				  - BUG: writeData() did not work correctly for ECOGIG OC26 moored data (spaces in filename?)

#				  - added support for patching coordinate system

#	Feb 16, 2016: - added transducer orientation to WBPens()

#				  - BUG: most WBPens() error messages used wrong file name

#	Feb 23, 2016: - changed WBRhdr() to use 2nd ensemble (with correct data-source id)

#	Feb 26, 2016: - added basic BT data to WBPens(); not BT_RL_* and BT_SIGNAL_STRENGTH

#	Feb 29, 2016: - LEAP DAY: actually got BT data patching to work

#	Jul 30, 2016: - BUG: incomplete last ensemble with garbage content was returned on reading

#						 WH300 data

#	Aug  5, 2016: - cosmetics

#	Aug 23, 2016: - added &clearEns()

#	Nov  9, 2016: - made WBRhdr() return undef on "empty" files

#	Nov 18, 2016: - BUG: ensNo was not reported correctly in format errors

#	Nov 23, 2016: - no longer set pitch/roll/heading to undef in clearEns()

#	Mar  7, 2016: - renamed round() to stop clashing with ANTSLIB

# FIRMWARE VERSIONS:

#	It appears that different firmware versions generate different file

#	structures. Currently (Sep 2005) these routines have been tested

#	with the following firmware versions (as reported by [listHdr]):

#

#------------------------------------------------------------

#	16.12	WH300 (1)				FSU 	A0304		53

# PD0 FILE FORMAT EXTENSIONS:

#

#	- file creator encoded in DATA_SOURCE_ID

#

#	- first ensemble uses default RDI DATA_SOURCE_ID because the LDEO_IX

#	  software assumes this

#

#

#	- DATA_SOURCE_ID = 0xA0 | PATCHED_MASK		produced by IMP+LADP 

#		PATCHED_MASK & 0x04:						pitch value has been patched

#		PATCHED_MASK & 0x02:						roll value has been patched

#		PATCHED_MASK & 0x01:						heading value has been patched

#			- PITCH & ROLL can be missing (0x8000 badval as in velocities)

#			- HEADING can be missing (0xF000 badval, as 0x8000 is valid 327.68 heading)

#

#	- DATA_SOURCE_ID = 0xE0 					produced by editPD0

# NOTES:

#	- RDI stores data in VAX/Intel byte order (little endian)

#	- the output data structure does not exactly mirror the file data

#	  structure; the header is not stored at all and the fixed leader

#	  data are not duplicated in every ensemble

#	- in the RDI files some fields that logically belong into the header

#	  or the fixed leader (e.g. BT_MIN_CORRELATION) appear in the 

#	  ensemble data --- these are not read on input

#	- the field names are generally unabbreviated except for

#	  BT (= Bottom Track), RL (= Reference Layer), MIN and MAX

#	- all arrays are 0-referenced, but the ensemble number is not!

#	- a list of filenames can be be passed to readData() so that

#	  files split onto several memory cards (typically .000 .001 &c)

#	  can be read --- not sure if this works, actually

#	- the RDI manuals are not entirely clear everywhere; I have made

#	  guesses in some cases, but they should not affect the main

#	  fields of interest

#	- some fields in the fixed leader are not really fixed in a LADCP

#	  setting (e.g. xducer orientation); I'v made an educated guess

#	  as to which fields to move to the ENS array

#	- all units except pressure are SI, i.e. in m and m/s

#	- I don't understand the ERROR_STATUS_WORD; here's what 3 different

#	  instruments returned:

#		0x88008180	LDEO uplooker (slave) during NBP0402 (Firmware 16.21)

#		0x00008180	LDEO downlooker (master) during NBP0402 (Firmware 16.21)

#	  According to the manual (January 2001 version) this would, for example,

#	  indicate power failures on both FSU and LDEO slave instruments...

# &readData() returns perl obj (ref to anonymous hash) with the following

# structure:

#

#	DATA_SOURCE_ID					scalar		0x7f (Workhorse, also DVL)

#	NUMBER_OF_DATA_TYPES			scalar		6 (no BT) or 7

#	ENSEMBLE_BYTES					scalar		?, number of bytes w/o checksum

#	HEADER_BYTES					scalar		?

#	FIXED_LEADER_BYTES				scalar		42 for BB150; 53 for WH300, 58 for WH600, 59 for WH300(Nash)

#	VARIABLE_LEADER_BYTES			scalar		?

#	CORRELATION_DATA_BYTES			scalar		?

#	ECHO_INTENSITY_DATA_BYTES		scalar		?

#	BT_PRESENT						bool		NUMBER_OF_DATA_TYPES == 7

#	BT_DATA_BYTES					scalar		undefined, ? if BT_PRESENT

#	CPU_FW_VER						scalar		0--255

#	CPU_FW_REV						scalar		0--255

#	BEAM_FREQUENCY					scalar		75, 150, 300, 600, 1200, 2400 [kHz]

#	CONCAVE_BEAM_PATTERN			bool		undefined, 1

#	SENSOR_CONFIG					scalar		1--3

#	XDUCER_HEAD_ATTACHED			bool		undefined, 1

#	BEAM_ANGLE						scalar		15,20,30,undefined=other [deg]

#	N_DEMODS						scalar		2--3(???),undefined=n/a

#	N_BINS							scalar		1--128

#	PINGS_PER_ENSEMBLE				scalar		0--16384

#	BIN_LENGTH						scalar		0.01--64 [m]

#	BLANKING_DISTANCE				scalar		0-99.99 [m]

#	N_CODE_REPETITIONS				scalar		0--255

#	MIN_PERCENT_GOOD				scalar		1--100 [%]

#	MAX_ERROR_VELOCITY				scalar		0--5 [m/s]

#	TIME_BETWEEN_PINGS				scalar		0--? [s]

#	BEAM_COORDINATES				bool		undefined,1

#	INSTRUMENT_COORDINATES			bool		undefined,1

#	SHIP_COORDINATES				bool		undefined,1

#	EARTH_COORDINATES				bool		undefined,1

#	USE_3_BEAM_ON_LOW_CORR			bool		undefined,1

#	CALCULATE_SPEED_OF_SOUND		bool		undefined,1

#	USE_PITCH_SENSOR				bool		undefined,1

#	USE_TEMPERATURE_SENSOR			bool		undefined,1

#	SPEED_OF_SOUND_CALCULATED		bool		undefined,1

#	PRESSURE_SENSOR_AVAILABLE		bool		undefined,1

#	COMPASS_AVAILABLE				bool		undefined,1

#	PITCH_SENSOR_AVAILABLE			bool		undefined,1

#	ROLL_SENSOR_AVAILABLE			bool		undefined,1

#	CONDUCTIVITY_SENSOR_AVAILABLE	bool		undefined,1

#	TEMPERATURE_SENSOR_AVAILABLE	bool		undefined,1

#	TRANSMITTED_PULSE_LENGTH		scalar		0--655.35 [m]

#	RL_FIRST_BIN					scalar		1--128

#	FALSE_TARGET_THRESHOLD			scalar		0--254, undefined=disabled

#	TRANSMIT_LAG_DISTANCE			scalar		0--655.35 [m]

#	CPU_SERIAL_NUMBER				scalar		undefined, 0--65535 if WH300

#	NARROW_BANDWIDTH				bool		undefined,1 (only set if WH300)

#	WIDE_BANDWIDTH					bool		undefined,1 (only set if WH300)

#	TRANSMIT_POWER					scalar		undefined, 0--255(high) if WH300

#	BT_PINGS_PER_ENSEMBLE			scalar		0--999

#	BT_DELAY_BEFORE_REACQUIRE		scalar		0--999

#	BT_MIN_CORRELATION				scalar		0--255

#	BT_MIN_EVAL_AMPLITUDE			scalar		0--255

#	BT_MAX_ERROR_VELOCITY			scalar		0--5 [m/s], undef=not screened

#	BT_RL_MIN_SIZE					scalar		0--99.9 [m]

#	BT_RL_NEAR						scalar		0--999.9 [m]

#	BT_RL_FAR						scalar		0--999.9 [m]

#	BT_MAX_TRACKING_DEPTH			scalar		8--999.9 [m]

#		XDUCER_FACING_UP			bool		undefined, 1

#		XDUCER_FACING_DOWN			bool		undefined, 1

#		N_BEAMS_USED				scalar		3,4,5(?)

#		NUMBER						scalar		1--16777215

#		SPEED_OF_SOUND				scalar		1400--1600 [m/s]

#		XDUCER_DEPTH				scalar		0.1--999.9 [m]

#		PITCH						scalar		-20.00-20.00 [deg] --- IMP EXTENSION: undef

#		ROLL						scalar		-20.00-20.00 [deg] --- IMP EXTENSION: undef

#		SALINITY					scalar		0-40 [psu]

#		MIN_PRE_PING_WAIT_TIME		scalar		? [s]

#		HEADING_STDDEV				scalar		0-180 [deg]

#		PITCH_STDDEV				scalar		0.0-20.0 [deg]

#		ADC_XMIT_CURRENT			scalar		0--255

#		ADC_XMIT_VOLTAGE			scalar		0--255

#		ADC_PRESSURE_PLUS			scalar		0--255

#		ADC_PRESSURE_MINUS			scalar		0--255

#		ADC_ATTITUDE_TEMPERATURE	scalar		0--255

#		ADC_ATTITUDE				scalar		0--255

#		ADC_CONTAMINATION			scalar		0--255

#		ERROR_STATUS_WORD			scalar		undefined, ? (only set if WH300)

#		PRESSURE					scalar		undefined, ?-? [dbar] (only set if WH300)

#		DATE						string		MM/DD/YYYY

#		YEAR						scalar		?

#		MONTH						scalar		1--12

#		TIME						string		HH:MM:SS.hh

#		HOUR						scalar		0--23

#		MINUTE						scalar		0--59

#		UNIX_TIME					scalar		0--?

#		SECNO						scalar		0--? (number of seconds since daystart)

#		DAYNO						double		fractional day number since start of current year (1.0 is midnight Jan 1st)

#		HIGH_GAIN					bool		1, undefined

#		LOW_GAIN					bool		1, undefined

#----------------------------------------------------------------------

# Time Conversion Subroutines

#----------------------------------------------------------------------

		  }

	  }

#----------------------------------------------------------------------

# Read Data

#----------------------------------------------------------------------

#----------------------------------------------------------------------

# skip to first valid ensemble (skip over initial garbage)

#----------------------------------------------------------------------

		}

	}

#----------------------------------------------------------------------

# readHeader(file_name,^dta) WBRhdr(^data)

#	- read header data

#	- also includes some data from 1st ens

#----------------------------------------------------------------------

sub readHeader(@)

{

	my($fn,$dta) = @_;

	$WBRcfn = $fn;

}

{

	my($start_ens,$buf,$hid,$did,$Ndt,$B,$W,$i,$dummy,$id,@WBRofs);

	my($B1,$B2,$B3,$B4,$B5,$B6,$B7,$W1,$W2,$W3,$W4,$W5);

	#--------------------

	# HEADER

	#--------------------

	skip_initial_trash();

	sysread(WBRF,$buf,6) == 6 || return undef;

	($hid,$did,$dta->{ENSEMBLE_BYTES},$dummy,$dta->{NUMBER_OF_DATA_TYPES})

		= unpack('CCvCC',$buf);

	$hid == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (hid)",$hid,0));

	$did == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (did)",$did,0));

	$start_ens = sysseek(WBRF,$dta->{ENSEMBLE_BYTES}-6+2,1) || return undef;

	sysread(WBRF,$buf,6) == 6 || return undef;

	($hid,$did,$dta->{ENSEMBLE_BYTES},$dummy,$dta->{NUMBER_OF_DATA_TYPES})

		= unpack('CCvCC',$buf);

	$hid == 0x7f || die(sprintf($FmtErr,$WBRcfn,"Header (hid2)",$hid,0));

	$dta->{DATA_SOURCE_ID} = $did;

	if ($did == 0x7f) {

		$dta->{PRODUCER} = 'TRDI ADCP';

	} elsif (($did&0xF0) == 0xA0) {

		$dta->{PRODUCER} = 'IMP+LADCP (Thurnherr software)';

	} elsif (($did&0xF0) == 0xE0) {

		$dta->{PRODUCER} = 'editPD0 (Thurnherr software)';

	} else {

		$dta->{PRODUCER} = sprintf('unknown (0x%02X)');

	}

	sysread(WBRF,$buf,2*$dta->{NUMBER_OF_DATA_TYPES})

		== 2*$dta->{NUMBER_OF_DATA_TYPES}

			|| die("$WBRcfn: $!");

	@WBRofs = unpack("v$dta->{NUMBER_OF_DATA_TYPES}",$buf);

#	for ($i=0; $i<$dta->{NUMBER_OF_DATA_TYPES}; $i++) {

#		printf(STDERR "\nWBRofs[$i] = %d",$WBRofs[$i]);

#	}

	sysseek(WBRF,$start_ens+$WBRofs[1],0) || die("$WBRcfn: $!");

	sysread(WBRF,$buf,2) == 2 || die("$WBRcfn: $!");

	$id = unpack('v',$buf);

	sysread(WBRF,$buf,2) == 2 || die("$WBRcfn: $!");