#!/usr/bin/perl

#======================================================================

#                    L A D C P _ W _ O C E A N 

#                    doc: Fri Dec 17 18:11:13 2010

#                    (c) 2010 A.M. Thurnherr

#======================================================================

# TODO:

#	- make upcast-flag valid for yoyo casts

#	- make diagnostic output 3-beam field work for Earth coordinates

#	- remove water-depth from BT code, which is not really used and a bit of an outlier

#	  because mean and stddev are used instead of median/mad

#	- read assumed ADCP soundspeed from data file, instead of assuming 1500m/s

#	? use instrument tilt in sidelobe editing?

$antsSummary = 'calculate vertical velocities from LADCP & CTD time series';

# HISTORY:

#	Dec 17, 2010: - created from [mergeCTD+LADCP]

#	Dec 18, 2010: - made to work

#	Dec 19, 2010: - improved considerably

#	Dec 20, 2010: - onward

#				  - BUG: depth-binning was off by 1 bin?!

#				  - added binning correction for instrument tilt

#	Dec 21, 2010: - added -h (seafloor depth)

#	Dec 22, 2010: - BUG: had not applied soundspeed-correction to w

#				  - debugged opt_d

#	Dec 23, 2010: - continued implementation of soundspeed corrections

#	Dec 24, 2010: - added winch_w, wave_w

#				  - removed beampair velocities from code

#	Dec 25, 2010: - adapted for surface-wave correction in terms of acceleration (CTD_w_t)

#				  - removed elapsed_mismatch

#				  - removed winch_w, wave_w

#	Dec 26, 2010: - made -p output layout independent of -x to avoid Makefile problems

#	Dec 30, 2010: - cleaned up some

#				  - folded-in backscatter calculation from shear method

#				  - folded-in BT calculation from shear method

#	Dec 31, 2010: - added weighted mean w profile to output

#	Jan  2, 2010: - BUG: BT_w.mad could bomb with division by 0

#				  - BUG: division by zero if no valid data

#	Jan  5, 2010: - adapted to allow for gaps in CTD time series

#	Feb 16, 2011: - cosmetics

#	Jun 22, 2011: - cosmetics

#	Jun 23, 2011: - disabled error on large rms reflr w

#				  - added -l

#				  - removed CTD headers from output

#	Jun 26, 2011: - added -u

#				  - changed package correction from acceleration to velocity, because of

#					Stan's Antarctic data set where accelerations are zero but package effects are

#					there

#	Jul  2, 2011: - increased tilt default to 15 degrees

#	Jul  3, 2011: - replaced old package-velocity correction -x code by new beamvel correction

#				  - removed -p

#				  - replaced -d by residual (diagnostics) output

#	Jul  4, 2011: - saved a snapshot in [Jul_04_2011]

#				  - removed output of binned profile (but not calculation code, which is required for residual)

#				  - BUG: firstGoodEns or lastGoodEns could end up in a reflr_w gap when valid LADCP data begin

#						 or end with CTD in water

#				  - moved rarely used option -s to -k

#				  - added -s)kip <ens> option

#				  - had to very very slightly relax an assertion (by 1e-10 seconds...)

#	Aug  3, 2011: - added -z)ap to truncate range, based on Stan's 2009 data set (017-019 most clearly)

#				    obs that final 2 velocities are outliers; could be due to color bar, should check with

#					residuals

#	Sep 22, 2011: - removed (commented out) CTD acceleration

#				  - added basename to Parameter File Matching

#	Sep 23, 2011: - cosmetics (lots)

#				  - added -d)

#	Oct  6, 2011: - removed old commented-out code (CTD acceleration & old -x)

#				  - renamed time-series field names

#				  - added uncorrected reflr w to time-series output

#	Oct 10, 2011: - BUG: LADCP_w in .tds output had not been soundspeed corrected

#				  - removed -x

#				  - added "false-positives" editing filter

#	Oct 11, 2011: - adapted to the new parameter file, removing a few options

#	Oct 12, 2011: - added surface-layer editing

#				  - made %min/max_depth/elapsed more accurate

#	Oct 13, 2011: - fiddled

#	Oct 14, 2011: - renamed .prof fields from .N to .nsamp

#				  - added %output_basename

#				  - removed all ">" from open to allow use in pipelines

#				  - made <run-label> argument optional

#				  - replaced stdout output by $w_out

#				  - renamed _out to out_; output_basename to out_basename

#	Oct 15, 2011: - added editWOutliers()

#				  - added step to remove single-ping noise

#	Oct 16, 2011: - BUG: ensemble and bin numbers in output files were off by 1

#	Oct 17, 2011: - added default run label

#				  - added out_BR

#				  - BUG: closed STDOUT caused problems with tee in plotting scripts

#				  - added %run_label

#				  - added PostProcess.sh

#	Oct 18, 2011: - BUG: %{min,max}_ens used ensemble indices, rather than numbers

#				  - disabled false-positives editing (removes too much good data on 2011_IWISE)

#	Oct 19, 2011: - BUG: time-series output included ensembles without valid w

#				  - increased post-process sleep time to 30s

#				  - added corr, Sv and echo_amplitude as new fields to w output

#	Oct 20, 2011: - added editFarBins()

#				  - added downcast flag to time-series output

#				  - added ensemble number to LADCP-time-series output

#	Oct 21, 2011: - BUG: Sv code was not enabled for uplooker data

#	Oct 24, 2011: - added code to copy %lat/%lon from CTD data

#				  - added %ADCP_freuquency, %ADCP_blanking_distance

#				  - %LADCP_bin_length => %ADCP_bin_length

#	Oct 26, 2011: - added $first_guess_timelag

#	Oct 27, 2011: - added w12, w34, ww (pitch/roll-weighted average) to full output

#				  - removed ww because it apparently does not improve the solutions

#				  - moved editTilt() to after time-series calculation to allow

#				    time matching to work even with strict tilt limit

#				  - added correctAttitude()

#	May 22, 2012: - adapted to ANTS V5

#	Oct 15, 2012: - added $edit_data_hook

#				  - HG COMMIT

#				  - separated dc/uc time lagging

#				  - removed support for TLhist

#	Oct 16, 2012: - added support for dc/uc only solutions

#	Oct 17, 2012: - renamed $edit_data_hook to $post_merge_hook

#				  - HG COMMIT

#	Feb 13, 2013: - BUG: CTD_neg_press_offset did not work for CTD time series with -ve starting depth

#	Mar 23, 2013: - cosmetics

#				  - HG COMMIT

#	Apr 22, 2013: - removed opt_? variable aliases

#	May  8, 2013: - replaced default run label (default to profiles) to make more readable directory structure

#	May 14, 2013: - opt_m => w_max_lim

#				  - BUG: time-series output had two messed-up fields

#	May 15, 2013: - added w12 and w34 (2-beam solutions) to profile output

#	May 16, 2013: - added CTD_w_tt to time-series & all-sample outputs

#			      - -a => -d (CTD depth offset)

#				  - implemented pressure-sensor acceleration correction (-a)

#				  - added re-gridding of full profile after ping-coherent error removal

#	Jun  5, 2013: - renamed $discard_velocities_from_beam to $bad_beam

#				  - BUG: $bad_beam did not discard BT_VELOCITY data

#	Jun  6, 2013: - BUG: error message had -a instead of -d

#	Sep  5, 2013: - BUG: w12/w34 do not work for earth-coordinate data, of course

#	Apr 17, 2014: - BUG: edit_tilt was never called when all recorded bins are valid

#	Apr 21, 2014: - updated comments

#	May 19, 2014: - began adding support for PPI filtering

#	May 20, 2014: - changed volume_scattering_coeff to Sv in output

#				  - added editPPI()

#	Jul  6, 2014: - BUG: nan water depth had been interpreted as known

#				  - BUG: sVelProf[] was not allowed to have any gaps

#	Jul  9, 2014: - BUG: Jul 6 bug fixes had been applied to older

#				  		 version

#				  - BUG: code meant to ensure gap-free svel profiles did not work correctly

#	Jul 12, 2014: - finally made output files executable

#	Apr  5, 2015: - added check for required software

#				  - BUG: removed dc/uc mean w fields from .prof again

#	Apr  7, 2015: - made LADCP_w callable from installation directory

#				  - BUG: -v default was wrong in usage message

#				  - replaced 'ens' in output files by 'ensemble'

#	Apr 16, 2015: - turned output specifies into lists (re-design of

#					plotting sub-system)

#				  - removed 30s sleep from PostProcess.sh call

#				  - disabled active output when ANTS are not available

#				  - removed /bin/ksh requirement

#				  - BUG: error messages were not reported in the log file

#				  - made seabed detection code more flexible

#				  - made reference-layer w_ocean warning more severe

#				  - BUG: info() did not write anything into logfile when format string was used

#				  - BUG: LADCP_lastBin was set 1 too low

#	Apr 20, 2015: - improved comments

#				  - improved diagnostic messages (warning on missing CTD temp)

#				  - added support for empirical backscatter correction

#	Apr 21, 2015: - improved screen log output

#	Apr 22, 2015: - BUG: $realLastGoodEns could be undefined, breaking plotting routines

#	Apr 24, 2015: - removed @ARGS

#				  - added %profile_id

#	May 13, 2015: - loosened "insufficient valid velocities" test for short casts

#	May 15, 2015: - added $min_valid_vels

#				  - BUG: LADCP_atbottom could be less than firstGoodEns

#	May 18, 2015: - added %LADCP_pulse_length, %dnXX

#	May 20, 2015: - added $PROF as newer alternative to $STN

#				  - replaced "require ProcessingParams" by "do ProcessingParams"

#	Jun 15, 2015: - clarified warning message

#	Jul 26, 2015: - added %output_grid_dz %output_grid_minsamp for use by [LADCP_w_regrid]

# 				  - began work on support for [libGMT.pl]

#				  - -v usage message had wrong default

#				  - added $outGrid_firstBin & $outGrid_lastBin

#	Jul 28, 2015: - added GMT plot support for all @out_ lists

#	Jul 29, 2015: - continue adaptation of code for new plotting system

#	Jul 30, 2015: - cosmetics

#	Sep  3, 2015: - changed out_w to out_wsamp

#	Sep 26, 2015: - replaced numberp($water_depth) by defined($water_depth) for consistency

#				  - implemented secondary sidelobe editing

#				  - allow for -h <filename>

#				  - allow $ID as alias for $PROF and $STN

#	Sep 27, 2015: - BUG: -h filename was broken when water_depth is not known

#	Oct 12, 2015: - upgraded missing water-depth warnings to L2

#				  - added -V)ersion

#                 - require ANTSlibs V6.2 for release

#   Oct 13, 2015: - adapted to [version.pl]

#	Nov 25, 2015: - made warning disappear on setting $ANTS_TOOLS_AVAILABLE

#	Nov 27, 2015: - changed RDI_BB_READ.pl to RDI_PD0_IO.pl

#	Jan  4, 2016: - decreased default vertical resolution to 20m

# HISTORY END

# CTD REQUIREMENTS

#	- elapsed		elapsed seconds; see note below

#	- depth

#	- ss			sound speed

#	- w				ddepth/dt

#	- temp			OPTIONAL; used for backscatter calculation

#	- %lat/%lon		OPTIONAL

# 2-BEAM SOLUTIONS

#	- for beam-coordinate data, two separate two-beam solutions (w12 & w34) are calculated

#	- discrepancies can be indicative of package wake effects (e.g. MIXET data)

#	- for earth-coordinate data, w12 and w34 are simply copies of w

# NUMERICAL OPTIONS

#	- the first option in the list cannot be numerical!

#	- if need be, use -v 1 as a dummy option

# ELAPSED TIMES

#	- there are 2 different elapsed times used in this program:

#		1) elapsed based on firstgoodens in the LADCP time series

#		2) CTD elapsed time

#	- CTD elapsed time does not have to start with zero!

#	- do not use the Seabird elapsed field, which is only reported to

#	  3 significant digits, causing significant jitter in dt; however,

#	  at least up to 2010 Seabird simply calculates elapsed time by 

#	  assuming a 24Hz sampling rate and no record drop; therefore,

#	  it is best to calculate elapsed time as %RECNO/24

#	- the elapsed field of the output is the elapsed time from the CTD

#	  file; this is required in order to be able to compare the times

#	  from the uplooker and downlooker-derived vertical velocity

#	  profiles

#	- the .TL files use the LADCP elapsed time in output; as a result,

#	  the DL/UL .TL files have different time axes!

#	- as a result, a profile only starts with elapsed==0 if the CTD

#	  is turned on when the LADCP is already in the water

# BINNED-PROFILE OUTPUT NOTES

#	- residuals are calculated with respect to down-/upcast medians

#	- w12 and w34 2-beam solutions are reported without considering

#	  -k (min samples)

#	- equivalence, assuming -o 10:

#		1) 004DL.prof dc_w depth

#		2) bindata -Sdowncast:1 -Fw.median,depth -n 20 depth 10 004DL.w

# VELOCITY AMBIGUITY ERRORS

#	- quite extensive tests with DIMES US2 station 146, which has a lot of

#	  ambiguity velocity errors, reveal that -m 1 catches those errors

#	  quite nicely

#	- even when the errors are not filtered with -m 1, they do not

#	  affect the w profiles, as long as the median bin values are used

($ANTS) 	  = (`which ANTSlib`   =~ m{^(.*)/[^/]*$});

($ADCP_TOOLS) = (`which mkProfile` =~ m{^(.*)/[^/]*$});

($WCALC) 	  = ($0                =~ m{^(.*)/[^/]*$});

$WCALC = '.' if ($WCALC eq '');

$ANTS_TOOLS_AVAILABLE = (`which list 2>/dev/null` ne '');

die("$0: Generic Mapping Tools (GMT) required but not found (bad \$PATH?)\n")

	unless (`which psxy` ne '');

die("$0: ANTSlib required but not found (bad \$PATH?)\n")

	unless ($ANTS ne '');

die("$0: ADCP Tools required but not found (bad \$PATH?)\n")

	unless ($ADCP_TOOLS ne '');

require "$WCALC/version.pl";

require "$WCALC/edit_data.pl";

require "$WCALC/time_series.pl";

require "$WCALC/time_lag.pl";

require "$WCALC/find_seabed.pl";

require "$WCALC/svel_corrections.pl";

require "$WCALC/acoustic_backscatter.pl";

require "$WCALC/bottom_tracking.pl";

require "$ANTS/ants.pl";

require "$ANTS/libstats.pl";

use IO::Handle;

my($TINY) = 1e-16;

#------

# Usage

#------

$antsParseHeader = 0;

&antsUsage('3:4a:b:c:e:g:h:i:k:m:n:o:p:qs:t:uv:Vw:x:',0,

	'[print software -V)ersion] [-v)erbosity <level[1]>]',

	'[-q)uick (no single-ping denoising)]',

    '[require -4)-beam solutions] [apply beamvel-m)ask <file> if it exists]',

	'[valid LADCP -b)ins <bin[2],bin[*]>',

	'[-c)orrelation <min[70]>] [-t)ilt <max[10deg]> [-e)rr-vel <max[0.1m/s]>]',

	'[-h water <depth|filename>]',

	'[max LADCP time-series -g)ap <length[60s]>]',

	'[-i)nitial CTD time offset <guestimate> [-u)se as final]]',

	'[calculate -n) <lags,lags[10,100]>] [lag -w)indow <sz,sz[240s,20s]>] [lag-p)iece <CTD_elapsed_min|+[,...]>]',

	'[require top-3) lags to account for <frac[0.6]> of all]',

	'[pressure-sensor -a)cceleration correction <residual/CTD_w_tt>]',

	'[-o)utput bin <resolution[20m]>] [-k) require <min[20]> samples]',

	'[e-x)ecute <perl-expr>]',

	'<profile-id> [run-label]');

	if ($opt_V) {

		printf(STDERR "+-------------------------+\n");

		printf(STDERR "| LADCP_w Software V%s   |\n",$VERSION);

		printf(STDERR "|(c) 2015- A.M. Thurnherr |\n");

		printf(STDERR "+-------------------------+\n");

		exit(0);

	}

	&antsUsageError() if ($opt_u && !defined($opt_i));

	&antsUsageError() unless (@ARGV==1 || @ARGV==2);

	&antsCardOpt(\$opt_s,0);												# skip # initial ensembles

	$opt_p = '+' unless defined($opt_p);									# separate uc/dc time lagging

	&antsFloatOpt(\$opt_a,1);												# pressure acceleration correction

	$ID = $PROF = $STN = &antsCardArg();									# station id ($STN for compatibility)

	if (@ARGV) {															# run label

		$RUN = $ARGV[0];

		shift;

	} else {

		$RUN = 'profiles';

	}

#-----------------------------

# Handle Processing Parameters

#-----------------------------

require "$WCALC/defaults.pl";											# load default/option parameters

do "$processing_param_file";											# load processing parameters

	croak("$0: cannot decode -b $opt_b\n")

    	unless (numberp($LADCP_firstBin) &&

        	    ($LADCP_lastBin eq '*' || numberp($LADCP_lastBin)));

@number_of_timelag_windows = split(',',$opt_n);

	croak("$0: cannot decode -n $opt_n\n")

		unless numberp($number_of_timelag_windows[0]) && numberp($number_of_timelag_windows[1]);

@length_of_timelag_windows = split(',',$opt_w);

	croak("$0: cannot decode -w $opt_w\n")

		unless numberp($length_of_timelag_windows[0]) && numberp($length_of_timelag_windows[1]);

croak("$0: \$out_basename undefined\n")									# all plotting routines use this

	unless defined($out_basename);

&antsAddParams('out_basename',$out_basename);

&antsAddParams('profile_id',$PROF,'run_label',$RUN);

#----------------------------------------------------------------------

# Screen Logging

#	- warning levels:

#		0 probably unimportant, e.g. nonsensical parameters that probably won't affect solution

#		1 may be somewhat important

#		2 important

#----------------------------------------------------------------------

open(LOGF,">$out_log") || croak("$out_log: $!\n")

	if defined($out_log);

LOGF->autoflush(1);

sub progress(@)

{

	printf(LOGF @_) if defined($out_log);

	printf(STDERR @_) if ($opt_v > 1);

}

sub info(@)

{

	print(LOGF "\t"),printf(LOGF @_) if defined($out_log);

	printf(STDERR @_)  if ($opt_v > 1);

}

sub warning(@)

{

	my($lvl,@msg) = @_;

	if (defined($out_log)) {

		print(LOGF "\nWARNING (L$lvl): ");

		printf(LOGF @msg);

		print(LOGF "\n");

	}

	return if ($opt_v == 0);

	if ($opt_v == 1) {

		print(STDERR "$LADCP_file: WARNING (L$lvl): ");

		printf(STDERR @msg);

	} else {

		print(STDERR "\n-------------\nWARNING (L$lvl): ");

		printf(STDERR @msg);

		print(STDERR "-------------\n\n")

	}

}

sub error($)

{

	print(LOGF "ABORT: @_") if defined($out_log);

	croak("ABORT: @_");

}

sub debugmsg(@)

{ printf(STDERR @_) if ($opt_v > 2); }

#----------------

# Read LADCP data

#----------------

progress("Reading LADCP data from <$LADCP_file>...\n");

readData($LADCP_file,\%LADCP);

progress("\t%d ensembles\n",scalar(@{$LADCP{ENSEMBLE}}));

error("$LADCP_file: not enough LADCP bins ($LADCP{N_BINS}) for choice of -r\n")

	unless ($LADCP{N_BINS} >= $refLr_lastBin);

error("$0: first reference-layer bin outside valid range\n")

	unless ($refLr_firstBin>=1 && $refLr_firstBin<=$LADCP{N_BINS});

error("$0: last reference-layer bin outside valid range\n")

	unless ($refLr_lastBin>=1 && $refLr_lastBin<=$LADCP{N_BINS});

error("$0: first reference-layer bin > last reference-layer bin\n")

	unless ($refLr_firstBin <= $refLr_lastBin);

error("$0: first valid LADCP bin outside valid range\n")

	unless ($LADCP_firstBin>=1 && $LADCP_firstBin<=$LADCP{N_BINS});

error("$0: last valid LADCP bin outside valid range\n")

	unless ($LADCP_lastBin>=1 && $LADCP_lastBin<=$LADCP{N_BINS});

error("$0: first valid LADCP bin > last valid LADCP bin\n")

	unless ($LADCP_firstBin <= $LADCP_lastBin);

warning(0,"first reference-layer bin < first valid LADCP bin\n")

	unless ($refLr_firstBin >= $LADCP_firstBin);

warning(0,"last reference-layer bin > last valid LADCP bin\n")

	unless ($refLr_lastBin <= $LADCP_lastBin);

warning(1,"if at all, bin 1 should not be used for short blank-after-transmit values\n")

	if ($LADCP{BLANKING_DISTANCE}<$LADCP{BIN_LENGTH} && $refLr_firstBin==1);

&antsAddParams('ADCP_bin_length',$LADCP{BIN_LENGTH},

			   'ADCP_pulse_length',$LADCP{TRANSMITTED_PULSE_LENGTH},

			   'ADCP_frequency',$LADCP{BEAM_FREQUENCY},

			   'ADCP_blanking_distance',$LADCP{BLANKING_DISTANCE});

#------------------------------------------------------------

# Edit beam-velocity data

#	0) beam-vel mask on -m

#	1) correlation threshold

#------------------------------------------------------------

if ($LADCP{BEAM_COORDINATES}) {

	progress("Editing beam-velocity data...\n");

	if (defined($opt_m) && -r $opt_m) {

		progress("\tmasking beam velocities according to $opt_m:");

		my($nee) = 0;

		open(BVM,$opt_m) || die("$opt_m: $!\n");

		while (<BVM>) {

			s/#.*//;

			s/^\s*$//;

			next if ($_ eq '');

			my($fe,$te,$db) = split;

			die("$opt_m: cannot decode $_\n")

				unless (numberp($fe) && numberp($te) && $te>=$fe && $db>=1 && $db<=4);

			die("$0: assertion failed")