#!/usr/bin/perl

#======================================================================

#                    L A D C P _ W _ O C E A N 

#                    doc: Fri Dec 17 18:11:13 2010

#                    (c) 2010 A.M. Thurnherr

#======================================================================

# TODO:

#   ! plots:

#       - avoid over-plotting axis labels

#       - allow for different nsamp magnitudes

#       - add "dc" "uc" labels

#       - add seabed to eps.VKE profile

#       - add seabed to LADCP_w_postproc .wprof output

#   ! use instrument tilt in sidelobe editing

#   - worry about water-depth differences (disabled warning) 

#   - make upcast-flag valid for yoyo casts

#   - make diagnostic output 3-beam field work for Earth coordinates

$antsSummary = 'calculate vertical velocities from LADCP & CTD time series';

# HISTORY:

#   Dec 17, 2010: - created from [mergeCTD+LADCP]

#   Dec 18, 2010: - made to work

#   Dec 19, 2010: - improved considerably

#   Dec 20, 2010: - onward

#                 - BUG: depth-binning was off by 1 bin?!

#                 - added binning correction for instrument tilt

#   Dec 21, 2010: - added -h (seafloor depth)

#   Dec 22, 2010: - BUG: had not applied soundspeed-correction to w

#                 - debugged opt_d

#   Dec 23, 2010: - continued implementation of soundspeed corrections

#   Dec 24, 2010: - added winch_w, wave_w

#                 - removed beampair velocities from code

#   Dec 25, 2010: - adapted for surface-wave correction in terms of acceleration (CTD_w_t)

#                 - removed elapsed_mismatch

#                 - removed winch_w, wave_w

#   Dec 26, 2010: - made -p output layout independent of -x to avoid Makefile problems

#   Dec 30, 2010: - cleaned up some

#                 - folded-in backscatter calculation from shear method

#                 - folded-in BT calculation from shear method

#   Dec 31, 2010: - added weighted mean w profile to output

#   Jan  2, 2010: - BUG: BT_w.mad could bomb with division by 0

#                 - BUG: division by zero if no valid data

#   Jan  5, 2010: - adapted to allow for gaps in CTD time series

#   Feb 16, 2011: - cosmetics

#   Jun 22, 2011: - cosmetics

#   Jun 23, 2011: - disabled error on large rms reflr w

#                 - added -l

#                 - removed CTD headers from output

#   Jun 26, 2011: - added -u

#                 - changed package correction from acceleration to velocity, because of

#                   Stan's Antarctic data set where accelerations are zero but package effects are

#                   there

#   Jul  2, 2011: - increased tilt default to 15 degrees

#   Jul  3, 2011: - replaced old package-velocity correction -x code by new beamvel correction

#                 - removed -p

#                 - replaced -d by residual (diagnostics) output

#   Jul  4, 2011: - saved a snapshot in [Jul_04_2011]

#                 - removed output of binned profile (but not calculation code, which is required for residual)

#                 - BUG: firstGoodEns or lastGoodEns could end up in a reflr_w gap when valid LADCP data begin

#                        or end with CTD in water

#                 - moved rarely used option -s to -k

#                 - added -s)kip <ens> option

#                 - had to very very slightly relax an assertion (by 1e-10 seconds...)

#   Aug  3, 2011: - added -z)ap to truncate range, based on Stan's 2009 data set (017-019 most clearly)

#                   obs that final 2 velocities are outliers; could be due to color bar, should check with

#                   residuals

#   Sep 22, 2011: - removed (commented out) CTD acceleration

#                 - added basename to Parameter File Matching

#   Sep 23, 2011: - cosmetics (lots)

#                 - added -d)

#   Oct  6, 2011: - removed old commented-out code (CTD acceleration & old -x)

#                 - renamed time-series field names

#                 - added uncorrected reflr w to time-series output

#   Oct 10, 2011: - BUG: LADCP_w in .tds output had not been soundspeed corrected

#                 - removed -x

#                 - added "false-positives" editing filter

#   Oct 11, 2011: - adapted to the new parameter file, removing a few options

#   Oct 12, 2011: - added surface-layer editing

#                 - made %min/max_depth/elapsed more accurate

#   Oct 13, 2011: - fiddled

#   Oct 14, 2011: - renamed .prof fields from .N to .nsamp

#                 - added %output_basename

#                 - removed all ">" from open to allow use in pipelines

#                 - made <run-label> argument optional

#                 - replaced stdout output by $w_out

#                 - renamed _out to out_; output_basename to out_basename

#   Oct 15, 2011: - added editWOutliers()

#                 - added step to remove single-ping noise

#   Oct 16, 2011: - BUG: ensemble and bin numbers in output files were off by 1

#   Oct 17, 2011: - added default run label

#                 - added out_BR

#                 - BUG: closed STDOUT caused problems with tee in plotting scripts

#                 - added %run_label

#                 - added PostProcess.sh

#   Oct 18, 2011: - BUG: %{min,max}_ens used ensemble indices, rather than numbers

#                 - disabled false-positives editing (removes too much good data on 2011_IWISE)

#   Oct 19, 2011: - BUG: time-series output included ensembles without valid w

#                 - increased post-process sleep time to 30s

#                 - added corr, Sv and echo_amplitude as new fields to w output

#   Oct 20, 2011: - added editFarBins()

#                 - added downcast flag to time-series output

#                 - added ensemble number to LADCP-time-series output

#   Oct 21, 2011: - BUG: Sv code was not enabled for uplooker data

#   Oct 24, 2011: - added code to copy %lat/%lon from CTD data

#                 - added %ADCP_freuquency, %ADCP_blanking_distance

#                 - %LADCP_bin_length => %ADCP_bin_length

#   Oct 26, 2011: - added $first_guess_timelag

#   Oct 27, 2011: - added w12, w34, ww (pitch/roll-weighted average) to full output

#                 - removed ww because it apparently does not improve the solutions

#                 - moved editTilt() to after time-series calculation to allow

#                   time matching to work even with strict tilt limit

#                 - added correctAttitude()

#   May 22, 2012: - adapted to ANTS V5

#   Oct 15, 2012: - added $edit_data_hook

#                 - HG COMMIT

#                 - separated dc/uc time lagging

#                 - removed support for TLhist

#   Oct 16, 2012: - added support for dc/uc only solutions

#   Oct 17, 2012: - renamed $edit_data_hook to $post_merge_hook

#                 - HG COMMIT

#   Feb 13, 2013: - BUG: CTD_neg_press_offset did not work for CTD time series with -ve starting depth

#   Mar 23, 2013: - cosmetics

#                 - HG COMMIT

#   Apr 22, 2013: - removed opt_? variable aliases

#   May  8, 2013: - replaced default run label (default to profiles) to make more readable directory structure

#   May 14, 2013: - opt_m => w_max_lim

#                 - BUG: time-series output had two messed-up fields

#   May 15, 2013: - added w12 and w34 (2-beam solutions) to profile output

#   May 16, 2013: - added CTD_w_tt to time-series & all-sample outputs

#                 - -a => -d (CTD depth offset)

#                 - implemented pressure-sensor acceleration correction (-a)

#                 - added re-gridding of full profile after ping-coherent error removal

#   Jun  5, 2013: - renamed $discard_velocities_from_beam to $bad_beam

#                 - BUG: $bad_beam did not discard BT_VELOCITY data

#   Jun  6, 2013: - BUG: error message had -a instead of -d

#   Sep  5, 2013: - BUG: w12/w34 do not work for earth-coordinate data, of course

#   Apr 17, 2014: - BUG: edit_tilt was never called when all recorded bins are valid

#   Apr 21, 2014: - updated comments

#   May 19, 2014: - began adding support for PPI filtering

#   May 20, 2014: - changed volume_scattering_coeff to Sv in output

#                 - added editPPI()

#   Jul  6, 2014: - BUG: nan water depth had been interpreted as known

#                 - BUG: sVelProf[] was not allowed to have any gaps

#   Jul  9, 2014: - BUG: Jul 6 bug fixes had been applied to older

#                        version

#                 - BUG: code meant to ensure gap-free svel profiles did not work correctly

#   Jul 12, 2014: - finally made output files executable

#   Apr  5, 2015: - added check for required software

#                 - BUG: removed dc/uc mean w fields from .prof again

#   Apr  7, 2015: - made LADCP_w callable from installation directory

#                 - BUG: -v default was wrong in usage message

#                 - replaced 'ens' in output files by 'ensemble'

#   Apr 16, 2015: - turned output specifies into lists (re-design of

#                   plotting sub-system)

#                 - removed 30s sleep from PostProcess.sh call

#                 - disabled active output when ANTS are not available

#                 - removed /bin/ksh requirement

#                 - BUG: error messages were not reported in the log file

#                 - made seabed detection code more flexible

#                 - made reference-layer w_ocean warning more severe

#                 - BUG: info() did not write anything into logfile when format string was used

#                 - BUG: LADCP_lastBin was set 1 too low

#   Apr 20, 2015: - improved comments

#                 - improved diagnostic messages (warning on missing CTD temp)

#                 - added support for empirical backscatter correction

#   Apr 21, 2015: - improved screen log output

#   Apr 22, 2015: - BUG: $realLastGoodEns could be undefined, breaking plotting routines

#   Apr 24, 2015: - removed @ARGS

#                 - added %profile_id

#   May 13, 2015: - loosened "insufficient valid velocities" test for short casts

#   May 15, 2015: - added $min_valid_vels

#                 - BUG: LADCP_atbottom could be less than firstGoodEns

#   May 18, 2015: - added %LADCP_pulse_length, %dnXX

#   May 20, 2015: - added $PROF as newer alternative to $STN

#                 - replaced "require ProcessingParams" by "do ProcessingParams"

#   Jun 15, 2015: - clarified warning message

#   Jul 26, 2015: - added %output_grid_dz %output_grid_minsamp for use by [LADCP_w_regrid]

#                 - began work on support for [libGMT.pl]

#                 - -v usage message had wrong default

#                 - added $outGrid_firstBin & $outGrid_lastBin

#   Jul 28, 2015: - added GMT plot support for all @out_ lists

#   Jul 29, 2015: - continue adaptation of code for new plotting system

#   Jul 30, 2015: - cosmetics

#   Sep  3, 2015: - changed out_w to out_wsamp

#   Sep 26, 2015: - replaced numberp($water_depth) by defined($water_depth) for consistency

#                 - implemented secondary sidelobe editing

#                 - allow for -h <filename>

#                 - allow $ID as alias for $PROF and $STN

#   Sep 27, 2015: - BUG: -h filename was broken when water_depth is not known

#   Oct 12, 2015: - upgraded missing water-depth warnings to L2

#                 - added -V)ersion

#                 - require ANTSlibs V6.2 for release

#   Oct 13, 2015: - adapted to [version.pl]

#   Nov 25, 2015: - made warning disappear on setting $ANTS_TOOLS_AVAILABLE

#   Nov 27, 2015: - changed RDI_BB_READ.pl to RDI_PD0_IO.pl

#   Jan  4, 2016: - decreased default vertical resolution to 20m

#   Jan  5, 2016: - adapted to [ADCP_tools_lib.pl]

#   Jan 22, 2016: - updated for improved mean_residuals plot

#                 - added per-bin residual quality check

#   Jan 25, 2016: - added antsAddParams() with version number

#   Jan 26, 2016: - added %processing_params, many others

#                 - expunged -d

#                 - implemented outGrid_firstBin eq '*' (also lastBin)

#   Jan 27, 2016: - BUG: outGrid_lastBin eq '*' did not work

#                 - removed large ref-lr w warning

#   Feb 14, 2016: - fiddled with ping-coherent residuals code, fixing

#                   0-2 potential (minor?) bugs related to outGrid_{first,last}Bin;

#                   output of new code checked against old code => identical

#   Feb 16, 2016: - BUG: per-bin-residual QC could cause division by zero

#   Feb 19, 2016: - added -l (disable time-lag filtering)

#   Mar  7, 2016: - added error message when -h is neither number nor file

#                 - BUG: -ve depth error message referred to obsolete -d

#                 - BUG: dn field name did not use zero filling for year number

#   Mar  8, 2016: - removed L0 water-depth-difference warning

#                 - added test for 1500m/s sound speed

#   Mar  9, 2016: - added hab field to .wprof output

#   Mar 13, 2016: - cleaned up warnings created before LADCP_file is defined

#                 - added sanity checks and warnings

#   Mar 17, 2016: - added {dc,uc}_rms_{tilt,w_pkg}

#                 - replaced a couple of **2 by &SQR()

#                 - replaced %ADCP_orientation values by DL & UL

#                 - {dc,uc}_rms_w_pkg => {dc,uc}_rms_accel_pkg for V1.2beta6

#   Mar 18, 2016: - added -l to %processing_options

#   Mar 24, 2016: - generalized plotting system (renamed variables)

#                 - made plotting system search current dir before LADCP_w dir

#                 - BUG: ProcessingParams syntax errors were not flagged

#                 - BUG: progress message in rarely used filter

#   Mar 25, 2016: - added -r)esidual rms filter

#                 - added -q -r to processing_options

#   Mar 26, 2016: - BUG: water_depth < CTD_maxdepth was allowed

#                 - round(CTD_depth)

#   Mar 29, 2016: - split [default_paths.pl] from [defaults.pl] to allow defaults

#                   to be read before usage

#				  - renamed _subdir to _dir variables

#				  - renamed post-process hook script to LADCP_w.PostProcess

#				  - added -r default (0.04m/s)

#   Mar 31, 2016: - changed version %PARAM

#	Apr  6, 2016: - BUG: GMT test looked for psxy, rather than gmt binary

#	May 12, 2106: - added v12, v34 to wsamp output

#	May 17, 2016: - added residual12, residual34 to wsamp output

#				  - changed pitch field in wsamp output to gimbal pitch

#				  - added w_t to .wsamp output for completeness

#	May 18, 2016: - replaced rms_tilt by mean_tilt

#				  - add (gimbal-)pitch, roll and residual{12,34} to wprof output

#				  - expunged super-confusing $realLastGoodEns

#				  - removed assumption of default 1500m/s sspd setting during acquisition

#				  - added w12, w34 for earth-coordinates

# HISTORY END

# CTD REQUIREMENTS

#	- elapsed		elapsed seconds; see note below

#	- depth

#	- ss			sound speed

#	- w				ddepth/dt

#	- temp			OPTIONAL; used for backscatter calculation

#	- %lat/%lon		OPTIONAL

# 2-BEAM SOLUTIONS

#	- both for beam- and earth-coordinate data, two separate two-beam

#	  solutions (w12 & w34) are calculated:

#	- the 2-beam solutions are largely independent of each other

#		=> key for detecting data problems, such as package wakes

# NUMERICAL OPTIONS

#	- the first option in the list cannot be numerical!

#	- if need be, use -v 1 as a dummy option

# ELAPSED TIMES

#	- there are 2 different elapsed times used in this program:

#		1) elapsed based on firstgoodens in the LADCP time series

#		2) CTD elapsed time

#	- CTD elapsed time does not have to start with zero!

#	- do not use the Seabird elapsed field, which is only reported to

#	  3 significant digits, causing significant jitter in dt; however,

#	  at least up to 2010 Seabird simply calculates elapsed time by 

#	  assuming a 24Hz sampling rate and no record drop; therefore,

#	  it is best to calculate elapsed time as %RECNO/24

#	- the elapsed field of the output is the elapsed time from the CTD

#	  file; this is required in order to be able to compare the times

#	  from the uplooker and downlooker-derived vertical velocity

#	  profiles

#	- the .TL files use the LADCP elapsed time in output; as a result,

#	  the DL/UL .TL files have different time axes!

#	- as a result, a profile only starts with elapsed==0 if the CTD

#	  is turned on when the LADCP is already in the water

# BINNED-PROFILE OUTPUT NOTES

#	- residuals are calculated with respect to down-/upcast medians

#	- w12 and w34 2-beam solutions are reported without considering

#	  -k (min samples)

#	- equivalence, assuming -o 10:

#		1) 004DL.prof dc_w depth

#		2) bindata -Sdowncast:1 -Fw.median,depth -n 20 depth 10 004DL.w

# VELOCITY AMBIGUITY ERRORS

#	- quite extensive tests with DIMES US2 station 146, which has a lot of

#	  ambiguity velocity errors, reveal that -m 1 catches those errors

#	  quite nicely

#	- even when the errors are not filtered with -m 1, they do not

#	  affect the w profiles, as long as the median bin values are used

($ANTS) 	  = (`which ANTSlib`   =~ m{^(.*)/[^/]*$});

($ADCP_TOOLS) = (`which mkProfile` =~ m{^(.*)/[^/]*$});

($WCALC) 	  = ($0                =~ m{^(.*)/[^/]*$});

$WCALC = '.' if ($WCALC eq '');

$ANTS_TOOLS_AVAILABLE = (`which list 2>/dev/null` ne '');

die("$0: Generic Mapping Tools (GMT) required but not found (bad \$PATH?)\n")

	unless (`which gmt` ne '');

die("$0: ANTSlib required but not found (bad \$PATH?)\n")

	unless ($ANTS ne '');

die("$0: ADCP Tools required but not found (bad \$PATH?)\n")

	unless ($ADCP_TOOLS ne '');

require "$WCALC/version.pl";

require "$WCALC/edit_data.pl";

require "$WCALC/time_series.pl";

require "$WCALC/time_lag.pl";

require "$WCALC/find_seabed.pl";

require "$WCALC/svel_corrections.pl";

require "$WCALC/acoustic_backscatter.pl";

require "$WCALC/bottom_tracking.pl";

require "$ANTS/ants.pl";

require "$ANTS/libstats.pl";

require "$ADCP_TOOLS/ADCP_tools_lib.pl";

&antsAddParams('LADCP_w_ocean::version',$VERSION);

use IO::Handle;

my($TINY) = 1e-16;

#------

# Usage

#------

require "$WCALC/defaults.pl";												# load default/option parameters

$antsParseHeader = 0;

&antsUsage('3:4a:b:c:e:g:h:i:k:lm:n:o:p:qr:s:t:uv:Vw:x:',0,

	"[print software -V)ersion] [-v)erbosity <level[$opt_v]>]",

	"[-q)uick (no single-ping denoising)]",

    "[require -4)-beam solutions] [apply beamvel-m)ask <file> if it exists]",

	"[valid LADCP -b)ins <bin,bin[$opt_b]>",

	"[-c)orrelation <min[$opt_c counts]>] [-t)ilt <max[$opt_t deg]> [-e)rr-vel <max[$opt_e m/s]>]",

	"[-r)esidual <rms.max[$opt_r m/s]>]",

	"[-h water <depth|filename>]",

	"[max LADCP time-series -g)ap <length[$opt_g s]>]",

	"[-i)nitial CTD time offset <guestimate> [-u)se as final]]",

	"[calculate -n) <lags,lags[$opt_n]>] [lag -w)indow <sz,sz[$opt_w s]>] [lag-p)iece <CTD_elapsed_min|+[,...]>]",

	"[require top-3) lags to account for <frac[$opt_3]> of all]",

	"[disable time-l)ag filtering]",

	"[pressure-sensor -a)cceleration-derivative correction <residual/CTD_w_tt>]",

	"[-o)utput bin <resolution[$opt_o m]>] [-k) require <min[$opt_k]> samples]",

	"[e-x)ecute <perl-expr>]",

	"<profile-id> [run-label]");

if ($opt_V) {

	printf(STDERR "+-------------------------+\n");

	printf(STDERR "| LADCP_w Software V%s	|\n",$VERSION);

	printf(STDERR "|(c) 2015- A.M. Thurnherr |\n");

	printf(STDERR "+-------------------------+\n");

	exit(0);

}

&antsUsageError() if ($opt_u && !defined($opt_i));

&antsUsageError() unless (@ARGV==1 || @ARGV==2);

&antsCardOpt(\$opt_s,0);												# skip # initial ensembles

$opt_p = '+' unless defined($opt_p);									# separate uc/dc time lagging

&antsFloatOpt(\$opt_a,1);												# pressure acceleration correction

$ID = $PROF = $STN = &antsCardArg();									# station id ($STN for compatibility)

if (@ARGV) {															# run label

	$RUN = $ARGV[0];

	shift;

} else {

	$RUN = 'profiles';

}

#-----------------------------

# Handle Processing Parameters

#-----------------------------

require "$WCALC/default_paths.pl";										# load default input/output paths

croak("$processing_param_file: $@")

	unless ($err = do "$processing_param_file");						# load processing parameters

$processing_options = "-k $opt_k -o $opt_o -c $opt_c -t $opt_t -e $opt_e -g $opt_g -3 $opt_3";

$processing_options .= " -i $opt_i" if defined($opt_i);

$processing_options .= " -r $opt_r" if defined($opt_r);

$processing_options .= ' -l' if defined($opt_l);

$processing_options .= ' -q' if defined($opt_q);

if (defined($opt_x)) {													# eval cmd-line expression to override anything

	$processing_options .= " -x $opt_x";

	eval($opt_x);

}

if ($opt_4) {															# disallow 3-beam solutions

	$processing_options .= " -4";

	$RDI_Coords::minValidVels = 4;

}

($LADCP_firstBin,$LADCP_lastBin) = split(',',$opt_b);					# select valid bins

	croak("$0: cannot decode -b $opt_b\n")

    	unless (numberp($LADCP_firstBin) &&

        	    ($LADCP_lastBin eq '*' || numberp($LADCP_lastBin)));

$processing_options .= " -b $opt_b";

$outGrid_firstBin = $LADCP_firstBin if ($outGrid_firstBin eq '*');		# default bins to use in gridded output

$outGrid_lastBin  = $LADCP_lastBin  if ($outGrid_lastBin  eq '*');		# NB: can still be '*'!!!

@number_of_timelag_windows = split(',',$opt_n);

	croak("$0: cannot decode -n $opt_n\n")

		unless numberp($number_of_timelag_windows[0]) && numberp($number_of_timelag_windows[1]);

$processing_options .= " -n $opt_n";

@length_of_timelag_windows = split(',',$opt_w);

	croak("$0: cannot decode -w $opt_w\n")

		unless numberp($length_of_timelag_windows[0]) && numberp($length_of_timelag_windows[1]);

$processing_options .= " -w $opt_w";

croak("$0: \$out_basename undefined\n")									# plotting routines use this to label the plots

	unless defined($out_basename);

&antsAddParams('processing_options',$processing_options);

&antsAddParams('out_basename',$out_basename);

&antsAddParams('profile_id',$PROF,'run_label',$RUN);

#----------------------------------------------------------------------

# Screen Logging

#	- warning levels:

#		0 probably unimportant, e.g. nonsensical parameters that probably won't affect solution

#		1 may be somewhat important

#		2 important

#----------------------------------------------------------------------

open(LOGF,">$out_log") || croak("$out_log: $!\n")

	if defined($out_log);

LOGF->autoflush(1);

sub progress(@)

{

	printf(LOGF @_) if defined($out_log);

	printf(STDERR @_) if ($opt_v > 1);

}

sub info(@)

{

	print(LOGF "\t"),printf(LOGF @_) if defined($out_log);

	printf(STDERR @_)  if ($opt_v > 1);

}

sub warning(@)

{

	my($lvl,@msg) = @_;

	if (defined($out_log)) {

		print(LOGF "\nWARNING (L$lvl): ");

		printf(LOGF @msg);

		print(LOGF "\n");

	}

	return if ($opt_v == 0);

	if ($opt_v == 1) {

		print(STDERR "$LADCP_file: ") if defined($LADCP_file);

		print(STDERR "WARNING (L$lvl): ");

		printf(STDERR @msg);

	} else {

		print(STDERR "\n-------------\nWARNING (L$lvl): ");

		printf(STDERR @msg);

		print(STDERR "-------------\n\n")

	}

}

sub error($)

{

	print(LOGF "ABORT: @_") if defined($out_log);

	croak("ABORT: @_");

}

sub debugmsg(@)

{ printf(STDERR @_) if ($opt_v > 2); }

#---------------------------

# Read LADCP data