# -*- coding: utf-8 -*-
"""
Created on Thu Oct 29 18:39:50 2015
@author: derrick
"""
# python 2 and 3 compatibility imports
from __future__ import print_function, absolute_import, unicode_literals
from __future__ import with_statement, nested_scopes, generators, division
import numpy as np
import obspy
import pandas as pd
import scipy
from scipy.cluster.hierarchy import linkage
from six import string_types
import detex
pd.options.mode.chained_assignment = None # mute setting copy warning
################ CLUSTERING FUNCTIONS AND CLASSES ################
[docs]def createCluster(CCreq=0.5,
fetch_arg='EventWaveForms',
filt=[1, 10, 2, True],
stationKey='StationKey.csv',
templateKey='TemplateKey.csv',
trim=[10, 120],
saveclust=True,
fileName='clust.pkl',
decimate=None,
dtype='double',
eventsOnAllStations=False,
enforceOrigin=False,
fillZeros=False,
phases=None):
"""
Function to create an instance of the ClusterStream class
Parameters
-------
CCreq : float, between 0 and 1
The minimum correlation coefficient for grouping waveforms.
0.0 results in all waveforms grouping together and 1.0 will not
form any groups.
fetch_arg : str or detex.getdata.DataFetcher instance
Fetch_arg of detex.getdata.quickFetch, see docs for details.
filt : list
A list of the required input parameters for the obspy bandpass
filter [freqmin, freqmax, corners, zerophase].
stationKey : str or pd.DataFrame
Path to the station key or DataFrame of station key.
templateKey : str or pd.DataFrame
Path to the template key or loaded template key in DataFrame.
trim : list
A list with seconds to trim from events with respect to the origin
time reported in the template key (or optionally a first arivial time,
see phases param for details). The default value of [10, 120]
means each event will be trimmed to only contain 10 seconds before
its origin time and 120 seconds after. The larger the values of this
argument the longer the computation time and chance of misalignment,
but values that are too small may trim out desired phases of the
waveform.
saveClust : bool
If true save the cluster object in the current working
directory. The name is controlled by the fileName parameter.
fileName : str
Path (or name) to save the clustering instance, only used
if saveClust is True.
decimate : int or None
A decimation factor to apply to all data in order to decrease run
time. Can be very useful if the the data are oversampled. For
example, if the data are sampled at 200 Hz but a 1 to 10 Hz
bandpass filter is applied it may be appropriate to apply a
decimation factor of 5 to bring the sampling rate down to 40 hz.
dytpe : str
An option to recast data type of the seismic data. Options are:
double- numpy float 64
single- numpy float 32, much faster and amenable to cuda GPU
processing, sacrifices precision.
eventsOnAllStations : bool
If True only use the events that occur on all stations, if
false let each station have an independent event list.
enforceOrigin : bool
If True make sure each trace starts at the reported origin time in
the template key. If not trim or merge with zeros. Required for
lag times to be meaningful for hypoDD input.
fillZeros : bool
If True fill zeros from trim[0] to trim[1]. Suggested for older
data or if only triggered data are available.
phases : None, str, or instance of DataFrame
If not None a path to phase picks or a DataFrame of phase picks that
will be used for trim values rather than referencing the origin time
of each event. See issue 25 on detex github page for why this
might be useful.
Returns
---------
An instance of the detex SSClustering class
"""
# Read in stationkey and template keys and check a few key parameters
stakey = detex.util.readKey(stationKey, key_type='station')
temkey = detex.util.readKey(templateKey, key_type='template')
_checkClusterInputs(filt, dtype, trim, decimate)
if phases is not None:
phases = detex.util.readKey(phases, "phases")
# get a data fetcher
fetcher = detex.getdata.quickFetch(fetch_arg, fillZeros=fillZeros)
# Intialize object DF that will be used to store cluster info
msg = 'Starting IO operations and data checks'
detex.log(__name__, msg, level='info', pri=True)
TRDF = _loadEvents(fetcher, filt, trim, stakey, temkey, decimate,
dtype, enforceOrigin=enforceOrigin, phases=phases)
if len(TRDF) < 1: # if no events survive
msg = ('No events survived pre-processing, check DataFetcher and event\
quality')
detex.log(__name__, msg, level='error')
# Prune event that do not occur on all stations if required
if eventsOnAllStations:
# get list of events that occur on all required stations
eventList = list(set.intersection(*[set(x) for x in TRDF.Events]))
eventList.sort()
if len(eventList) < 2:
msg = 'less than 2 events in population have required stations'
detex.log(__name__, msg, level='error')
# Test number of events in each station
if len(TRDF) < 1:
msg = 'No events survived preprocessing, examin input args and data'
detex.log(__name__, msg, level='error', pri=True)
# Loop through entries for each station, perform clustering
for ind, row in TRDF.iterrows():
msg = 'performing cluster analysis on ' + row.Station
detex.log(__name__, msg, level='info', pri=True)
if not eventsOnAllStations:
eventList = row.Events
if len(row.Events) < 2: # if only one event on this station skip it
msg = 'Less than 2 valid events on station ' + row.Station
detex.log(__name__, msg, level='warning', pri=True)
continue
DFcc, DFlag, DFsubsamp = _makeDFcclags(eventList, row)
TRDF.Lags[ind] = DFlag
TRDF.CCs[ind] = DFcc
TRDF.Subsamp[ind] = DFsubsamp
cx = np.array([])
cxdf = 1.0000001 - DFcc # get dissimilarities
# flatten ccs and remove nans
cx = _flatNoNan(cxdf)
link = linkage(cx) # get cluster linkage
TRDF.loc[ind, 'Link'] = link
# define columns to keep
colstk = ['Station', 'Link', 'CCs', 'Lags', 'Subsamp', 'Events', 'Stats']
trdf = TRDF[colstk]
eventListAll = list(set.union(*[set(x) for x in TRDF.Events]))
eventListAll.sort()
# try:
clust = detex.subspace.ClusterStream(trdf, temkey, stakey, fetcher,
eventListAll, CCreq, filt, decimate,
trim, fileName, eventsOnAllStations,
enforceOrigin)
if saveclust:
clust.write()
return clust
######################### SUBSPACE FUNCTIONS AND CLASSES #####################
[docs]def createSubSpace(Pf=10 ** -12, clust='clust.pkl', minEvents=2, dtype='double',
conDatFetcher=None):
"""
Function to create subspaces on all available stations based on the
clusters in Clustering object which will either be passed directly as the
keyword clust or will be loaded from the path in clustFile
Parameters
-----------
Pf : float
The probability of false detection as modeled by the statistical
framework in Harris 2006 Theory (eq 20, not yet supported)
Or by fitting a PDF to an empirical estimation of the null space
(similar to Wiechecki-Vergara 2001). Thresholds are not set until
calling the SVD function of the subspace stream class.
clust: str or instance of detex.subspace.ClusterStream
The path to a pickled instance of ClusterStream or an instance of
ClusterStream. Used in defining the subspaces.
minEvents : int
The Min number of events that must be in a cluster in order for a
subspace to be created from that cluster.
dtype : str ('single' or 'double')
The data type of the numpy arrays used in the detections. Options are:
single- a np.float32, slightly faster (~30%) less precise
double- a np.float64 (default)
conDatFetcher : None, str, or instance of detex.getdata.DataFetcher
Parameter to indicate how continuous data will be fetched in the newly
created instance of SubSpace. Descriptions are the three accepted types
are:
1. (None) If None is passed detex will try to deduce the appropriate
type of DataFetcher from the event datafetcher attached to cluster
instance.
2. (str) conDatFetcher is a string it will be passed to
detex.getdata.quickFetch function which expects a path to the
directory where the data are stored or a valid DataFetcher method.
See the docs of the quickFetch function in detex.getdata for more info.
3. (instance of detex.getdata.DataFetcher) If an instance of
detex.getdata.DataFetcher is passed then it will be used as the
continuous data fetcher.
Returns
-----------
An instance of the SubSpace class
Note
----------
Most of the parameters that define how to fetch seismic data, which events
and stations to use in the analysis, filter parameters, etc. are already
defined in the cluster (ClusterStream) instance.
"""
# Read in cluster instance
if isinstance(clust, string_types): # if no cluster object passed read a pickled one
cl = detex.util.loadClusters(clust)
elif isinstance(clust, detex.subspace.ClusterStream):
cl = clust
else:
msg = 'Invalid clust type, must be a path or ClusterStream instance.'
detex.log(__name__, msg, level='error', e=ValueError)
# Get info from cluster, load fetchers
temkey = cl.temkey
stakey = cl.stakey
efetcher = cl.fetcher
if isinstance(conDatFetcher, detex.getdata.DataFetcher):
cfetcher = conDatFetcher
elif isinstance(conDatFetcher, string_types):
cfetcher = detex.getdata.quickFetch(conDatFetcher)
elif conDatFetcher is None:
if efetcher.method == 'dir':
cfetcher = detex.getdata.quickFetch('ContinuousWaveForms')
else: # if not directory assume obspy client used for both
cfetcher = efetcher
# Load events into main dataframe to create subspaces
TRDF = _loadEvents(efetcher, cl.filt, cl.trim, stakey, temkey, cl.decimate,
dtype)
for ind, row in TRDF.iterrows(): # Fill in cluster info from cluster object
TRDF.loc[ind, 'Link'] = cl[row.Station].link
TRDF.loc[ind, 'Clust'] = cl[row.Station].clusts
# Start subspace construction
msg = 'Starting Subspace Construction'
detex.log(__name__, msg, pri=True)
ssDict = {} # dict to store subspaces in
for num, row in TRDF.iterrows(): # Loop through each station
staSS = _makeSSDF(row, minEvents)
if len(staSS) < 1: # if no clusters form on current station
msg = 'No events grouped into subspaces on %s' % row.Station
detex.log(__name__, msg, level='warning', pri=True)
continue
for sind, srow in staSS.iterrows(): # loop each cluster
eventList = srow.Events
# get correlation values from cl object
DFcc, DFlag = _getInfoFromClust(cl, srow)
staSS['Lags'][sind] = DFlag
staSS['CCs'][sind] = DFcc
cxdf = 1.0000001 - DFcc
cx = _flatNoNan(cxdf)
cx, cxdf = _ensureUnique(cx, cxdf) # ensure cc value is unique
lags = _flatNoNan(DFlag)
link = linkage(cx) # get cluster linkage
staSS.loc[sind, 'Link'] = link
# get lag times and align waveforms
CCtoLag = _makeCC2LagMap(cx, lags) # a map from cc to lag times
delays, dflink = _getDelays(link, CCtoLag, cx, lags, cxdf)
delayNP = -1 * np.min(delays)
delayDF = pd.DataFrame(delays + delayNP, columns=['SampleDelays'])
delayDF['Events'] = [eventList[x] for x in delayDF.index]
staSS['AlignedTD'][sind] = _alignTD(delayDF, srow)
ustimes = _updateStartTimes(srow, delayDF, temkey)
staSS['Stats'][sind] = ustimes # update Start Times
offsets = _getOffsetList(sind, srow, staSS)
offsetAr = [np.min(offsets), np.median(offsets), np.max(offsets)]
staSS['Offsets'][sind] = offsetAr
# Put output into subspaceDict
staOut = staSS.drop(['MPfd', 'MPtd', 'Link', 'Lags', 'CCs'], axis=1)
ssDict[row.Station] = staOut
# make a list of sngles to pass to subspace class
singDic = _makeSingleEventDict(cl, TRDF, temkey)
substream = detex.subspace.SubSpace(singDic, ssDict, cl, dtype, Pf,
cfetcher)
msg = "Finished CreateSubSpace call"
detex.log(__name__, msg, level='info', pri=True)
return substream
def _getInfoFromClust(cl, srow):
"""
get the DFcc dataframe and lags dataframe from values already stored in
cluster object to avoid recalculating them
"""
sta = srow.Station
cll = cl.trdf[cl.trdf.Station == sta].iloc[0]
odi = _makeEventListKey(srow.Events, cll.Events)
inds = odi[:-1]
cols = odi[1:]
DFlag = cll.Lags.loc[inds, cols]
DFlag.index = range(len(DFlag))
DFlag.columns = range(1, len(DFlag) + 1)
DFcc = cll.CCs.loc[inds, cols]
DFcc.index = range(len(DFlag))
DFcc.columns = range(1, len(DFlag) + 1)
return DFcc, DFlag
def _makeEventListKey(evelist1, evelist2):
"""
Make index key to make evelist1 to evelist2
"""
odi = [_fastWhere(x, evelist2) for num, x in enumerate(evelist1)]
return odi
def _fastWhere(eve, objs):
"""
like np.where but a bit faster because data checks are skipped
"""
an = next(nu for nu, obj in enumerate(objs) if eve == obj)
return an
def _getOffsetList(sind, srow, staSS):
"""
get list of offsets and return it
"""
return [staSS.loc[sind, 'Stats'][x]['offset'] for x in srow.Stats.keys()]
def _updateStartTimes(srow, delayDF, temkey):
"""
Update the starttimes to reflect the values trimed in alignement
"""
statsdict = srow.Stats
sdo = srow.Stats
for key in sdo.keys():
temtemkey = temkey.loc[temkey.NAME == key].iloc[0]
delaysamps = delayDF[delayDF.Events == key].iloc[0].SampleDelays
Nc = sdo[key]['Nc']
sr = sdo[key]['sampling_rate']
stime = sdo[key]['starttime']
stime_new = stime + delaysamps / (sr * Nc) # updated starttime to trim
statsdict[key]['starttime'] = stime_new
otime = obspy.UTCDateTime(temtemkey.TIME).timestamp # starttime
statsdict[key]['starttime'] = stime + delaysamps / (sr * Nc)
statsdict[key]['origintime'] = otime
statsdict[key]['magnitude'] = temtemkey.MAG
statsdict[key]['offset'] = stime_new - otime # predict offset time
return statsdict
def _makeDFcclags(eventList, row):
"""
Function to make correlation matrix and lag time matrix
"""
cols = np.arange(1, len(eventList))
indicies = np.arange(0, len(eventList) - 1)
DFcc = pd.DataFrame(columns=cols, index=indicies)
DFlag = pd.DataFrame(columns=cols, index=indicies)
DFsubsamp = pd.DataFrame(columns=cols, index=indicies)
# Loop over indicies and fill in cc and lags
for b in DFcc.index.values:
for c in range(b + 1, len(DFcc) + 1):
rev = 1 # if order is switched multiply lags by -1
mptd1 = row.loc['MPtd'][eventList[b]]
mptd2 = row.loc['MPtd'][eventList[c]]
mpfd1 = row.loc['MPfd'][eventList[b]]
mpfd2 = row.loc['MPfd'][eventList[c]]
Nc1 = row.loc['Channels'][eventList[b]]
Nc2 = row.loc['Channels'][eventList[c]]
maxcc, sampleLag, subsamp = _CCX2(mpfd1, mpfd2, mptd1, mptd2, Nc1,
Nc2)
DFcc.loc[b, c] = maxcc
DFlag.loc[b, c] = sampleLag
DFsubsamp.loc[b, c] = subsamp
return DFcc, DFlag * rev, DFsubsamp
def _subSamp(Ceval, ind):
"""
Method to estimate subsample time delays using cosine-fit interpolation
Cespedes, I., Huang, Y., Ophir, J. & Spratt, S.
Methods for estimation of sub-sample time delays of digitized echo signals.
Ultrason. Imaging 17, 142–171 (1995)
Returns
-------
The amount the sample should be shifted (float between -.5 and .5)
"""
# If max occurs at start or end of CC no extrapolation
if ind == 0 or ind == len(Ceval) - 1:
tau = 0.0
else:
cb4 = Ceval[ind - 1]
caf = Ceval[ind + 1]
cn = Ceval[ind]
alpha = np.arccos((cb4 + caf) / (2 * cn))
alsi = np.sin(alpha)
tau = -(np.arctan((cb4 - caf) / (2 * cn * alsi)) / alpha)
if abs(tau) > .5:
msg = ('subsample failing, more than .5 sample shift predicted')
detex.log(__name__, msg, level='Warning', pri=True)
return ind
return tau
def _CCX2(mpfd1, mpfd2, mptd1, mptd2, Nc1, Nc2):
"""
Function find max correlation coeficient and corresponding lag time
between 2 traces. fft should already have been calculated
"""
if len(Nc1) != len(Nc2): # make sure there are the same number of channels
msg = 'Number of Channels not equal, cannot perform correlation'
detex.log(__name__, msg, level='error')
Nc = len(Nc1) # Number of channels
if len(mptd1) != len(mptd2) or len(mpfd2) != len(mpfd1):
msg = 'Lengths not equal on multiplexed data, cannot correlate'
detex.log(__name__, msg, level='error')
n = len(mptd1)
trunc = n // (2 * Nc) - 1 # truncate value
# trunc = n - 1
# n = trunc + 1
mptd2Temp = mptd2.copy()
mptd2Temp = np.lib.pad(mptd2Temp, (n - 1, n - 1), str('constant'),
constant_values=(0, 0))
a = pd.rolling_mean(mptd2Temp, n)[n - 1:]
b = pd.rolling_std(mptd2Temp, n)[n - 1:]
b *= np.sqrt((n - 1.0) / n)
c = np.real(scipy.fftpack.ifft(np.multiply(np.conj(mpfd1), mpfd2)))
c1 = np.concatenate([c[-(n - 1):], c[:n]]) # swap end to start
# slice by # of channels as not to mix match chans in multplexed stream
result = ((c1 - mptd1.sum() * a) / (n * b * np.std(mptd1)))[Nc - 1::Nc]
result = result[trunc: -trunc]
try:
maxcc = np.nanmax(result)
mincc = np.nanmin(result)
maxind = np.nanargmax(result)
if maxcc > 1. or mincc < -1.: # if a inf is found in array
# this can happen if some of the waveforms have been zeroed out
result[(result > 1) | (result < -1)] = 0
maxcc = np.nanmax(result)
maxind = np.nanargmax(result)
except ValueError: # if fails skip
return 0.0, 0.0, 0.0
subsamp = _subSamp(result, maxind)
return maxcc, (maxind + 1 + trunc) * Nc - (n), subsamp
[docs]def fast_normcorr(t, s):
"""
fast normalized cc
"""
if len(t) > len(s): # switch t and s if t is larger than s
t, s = s, t
n = len(t)
nt = (t - np.mean(t)) / (np.std(t) * n)
sum_nt = nt.sum()
a = pd.rolling_mean(s, n)[n - 1:]
b = pd.rolling_std(s, n)[n - 1:]
b *= np.sqrt((n - 1.0) / n)
c = np.convolve(nt[::-1], s, mode="valid")
result = (c - sum_nt * a) / b
return result
def _alignTD(delayDF, srow):
"""
loop through delay Df and apply offsets to create alligned
arrays dictionary
"""
aligned = {}
# find the required length for each aligned stream
TDlengths = len(srow.MPtd[delayDF.Events[0]]) - max(delayDF.SampleDelays)
for ind, row in delayDF.iterrows():
orig = srow.MPtd[row.Events]
orig = orig[row.SampleDelays:]
orig = orig[:TDlengths]
aligned[row.Events] = orig
if len(orig) == 0:
msg = ('Alignment of multiplexed stream failing on %s, \
try raising ccreq or widenning trim window' % srow.Station)
msg2 = _idAlignProblems(delayDF)
detex.log(__name__, msg + msg2, level='error')
return aligned
def _idAlignProblems(delayDF, m=7):
"""
Function that is called when alignment fails, trys to ID which events
are causing problems and append this info to the message that is sent to
the logger
"""
msg = ''
offsets = delayDF.SampleDelays
d = np.abs(offsets - np.median(offsets))
mdev = np.median(d)
s = d / mdev if mdev else 0.
offs = offsets[s > m] # events that are potentially causing problems
for ind, off in offs.iteritems():
msg += (('\nAlignment shift for event %s is an outlier '
'consider removing it') % delayDF.loc[ind].Events)
return msg
def _makeSingleEventDict(cl, TRDF, temkey):
"""
Make dict of dataframes for singles on each station
"""
singlesdict = {}
cols = [x for x in TRDF.columns if not x in ['Clust', 'Link', 'Lags', 'CCs']]
for num, row in TRDF.iterrows():
singleslist = [0] * len(cl[row.Station].singles) # init list
DF = pd.DataFrame(index=xrange(len(singleslist)), columns=cols)
if len(singleslist) < 1: # if no singles on this channel
break
DF['Name'] = str
DF['Offsets'] = list
for sn, sing in enumerate(singleslist):
# DF.Events[a[0]]=evlist\
evelist = [cl[row.Station].singles[sn]]
temtemkey = temkey.loc[temkey.NAME == evelist[0]].iloc[0]
DF["Station"][sn] = row.Station
DF["MPtd"][sn] = _trimDict(row, 'MPtd', evelist)
DF["MPfd"][sn] = _trimDict(row, 'MPfd', evelist)
DF["Stats"][sn] = _trimDict(row, 'Stats', evelist)
DF["Channels"][sn] = _trimDict(row, 'Channels', evelist)
otime = obspy.UTCDateTime(temtemkey.TIME).timestamp
stime = DF.Stats[sn][evelist[0]]['starttime']
DF.Stats[sn][evelist[0]]['origintime'] = otime
DF.Stats[sn][evelist[0]]['offset'] = stime - otime
DF.Stats[sn][evelist[0]]['magnitude'] = temtemkey.MAG
DF.Events[sn] = DF.MPtd[sn].keys()
DF.Name[sn] = 'SG%d' % sn
DF['SampleTrims'] = [{} for x in range(len(DF))]
DF['FAS'] = object
DF['Threshold'] = object
singlesdict[row.Station] = DF
return singlesdict
def _makeSSDF(row, minEvents):
"""
Function to change form of TRDF for subpace creation
"""
index = range(len(row.Clust))
columns = [x for x in row.index if x != 'Clust']
DF = pd.DataFrame(index=index, columns=columns)
DF['Name'] = ['SS%d' % x for x in range(len(DF))] # name subspaces
# Initialize columns for future use
DF['Events'] = object
DF['AlignedTD'] = object
DF['SVD'] = object
DF['UsedSVDKeys'] = object
DF['FracEnergy'] = object
DF['SVDdefined'] = False
DF['SampleTrims'] = [{} for x in range(len(DF))]
DF['Threshold'] = np.float
DF['SigDimRep'] = object
DF['FAS'] = object
DF['NumBasis'] = int
DF['Offsets'] = object
DF['Stats'] = object
DF['MPtd'] = object
DF['MPfd'] = object
DF['Channels'] = object
DF['Station'] = row.Station
DF = DF.astype(object)
for ind, row2 in DF.iterrows():
evelist = row.Clust[ind]
evelist.sort()
DF['Events'][ind] = evelist
DF['numEvents'][ind] = len(evelist)
DF['MPtd'][ind] = _trimDict(row, 'MPtd', evelist)
DF['MPfd'][ind] = _trimDict(row, 'MPfd', evelist)
DF['Stats'][ind] = _trimDict(row, 'Stats', evelist)
DF['Channels'][ind] = _trimDict(row, 'Channels', evelist)
# only keep subspaces that meet min req, dont renumber
DF = DF[[len(x) >= minEvents for x in DF.Events]]
# DF.reset_index(drop=True, inplace=True)
return DF
def _trimDict(row, column, evelist):
"""
function used to get only desired values form dictionary
"""
temdict = {k: row[column].get(k, None) for k in evelist}
dictout = {k: v for k, v in temdict.items() if not v is None}
return dictout
############## Shared Subspace and Cluster Functions #####################
def _loadEvents(fetcher, filt, trim, stakey, temkey, decimate, dtype,
enforceOrigin=False, phases=None):
"""
Initialize TRDF, a container for a great many things including
event templates, multiplexed data, obspy traces etc.
"""
columns = ['Events', 'MPtd', 'MPfd', 'Channels', 'Stats', 'Link', 'Clust',
'Lags', 'Subsamp', 'CCs', 'numEvents']
TRDF = pd.DataFrame(columns=columns)
stanets = stakey.NETWORK + '.' + stakey.STATION
TRDF['Station'] = stanets
TRDF['Keep'] = True
TRDF = TRDF.astype(object) # cast as objects so pandas can be abused ;)
# Make list in data frame that shows which stations are in each event
# Load streams into dataframe to call later
for ind, row in TRDF.iterrows():
TRDF['MPtd'][ind] = {}
TRDF['MPfd'][ind] = {}
sts, eves, chans, stats = _loadStream(fetcher, filt, trim, decimate,
row.Station, dtype, temkey,
stakey, enforceOrigin,
phases=phases)
TRDF['Events'][ind] = eves
TRDF['Channels'][ind] = chans
TRDF['Stats'][ind] = stats
# Make sure some events are good for given station
if not isinstance(TRDF['Events'][ind], list):
TRDF.loc[ind, 'Keep'] = False
continue
TRDF.loc[ind, 'numEvents'] = len(TRDF.loc[ind, 'Events'])
# get multiplexed time domain and freq. domain arrays
TRDF = _getTimeDomainWFs(TRDF, row, ind, sts, eves)
TRDF = _testStreamLengths(TRDF, row, ind)
TRDF = _getFreqDomain(TRDF, row, ind)
TRDF = TRDF[TRDF.Keep]
TRDF.sort_values(by='Station', inplace=True)
TRDF.reset_index(inplace=True, drop=True)
return TRDF
def _getTimeDomainWFs(TRDF, row, ind, sts, eves):
for key in eves: # loop each event
Nc = TRDF.loc[ind, 'Stats'][key]['Nc']
mp = multiplex(sts[key], Nc) # multiplexed time domain
st = sts[key] # current stream
TRDF.loc[ind, 'MPtd'][key] = mp
stu = st[0].stats.starttime.timestamp # updated starttime
TRDF.loc[ind, 'Stats']['starttime'] = stu
return TRDF
def _getFreqDomain(TRDF, row, ind):
for key in row.Events: # loop each event
mp = TRDF.loc[ind, 'MPtd'][key]
reqlen = 2 * len(mp) # required length
reqlenbits = 2 ** reqlen.bit_length() # required length fd
mpfd = scipy.fftpack.fft(mp, n=reqlenbits)
TRDF.loc[ind, 'MPfd'][key] = mpfd
return TRDF
def _testStreamLengths(TRDF, row, ind):
lens = np.array([len(x) for x in row.MPtd.values()])
# trim to smallest length if within 90% of median, else kill key
le = np.min(lens[lens > np.median(lens) * .9])
keysToKill = [x for x in row.Events if len(row.MPtd[x]) < le]
# trim events slightly too small if any
for key in row.Events:
trimed = row.MPtd[key][:le]
TRDF.loc[ind, 'MPtd'][key] = trimed
# rest keys on TRDF
tmar = np.array(TRDF.Events[ind])
tk = [not x in keysToKill for x in TRDF.Events[ind]]
TRDF.Events[ind] = tmar[np.array(tk)]
for key in keysToKill:
msg = ('%s on %s is out of length tolerance, removing' %
(key, row.Station))
detex.log(__name__, msg, level='warn', pri=True)
TRDF.MPtd[ind].pop(key, None)
return TRDF
def _flatNoNan(df):
"""
Take a dataframe of pure numerics, flatten into 1d array and remove NaNs
"""
df.fillna(np.NAN, inplace=True)
ar = df.values.flatten()
return ar[~np.isnan(ar)]
def _getDelays(link, CCtoLag, cx, lags, cxdf):
N = len(link) # Lumber of events
# append cluster numbers to link array
linkup = np.append(link, np.arange(N + 1, 2 * N + 1).reshape(N, 1), 1)
clustDict = _getClustDict(linkup, len(linkup))
cols = ['i1', 'i2', 'cc', 'num', 'clust']
dflink = pd.DataFrame(linkup, columns=cols).astype(object)
if len(dflink) > 0:
dflink['II'] = list
dflink['ev1'] = 0
dflink['ev2'] = 0
dflink = dflink.astype(object, copy=False)
for ind, row in dflink.iterrows():
ii1 = clustDict[int(row.i1)].tolist()
ii2 = clustDict[int(row.i2)].tolist()
dflink.loc[ind, 'II'] = ii1 + ii2
# get a dataframe with only index as ev1 and column as ev2
tempdf = cxdf[cxdf == row.cc].dropna(how='all').dropna(axis=1)
dflink.loc[ind, 'ev1'] = tempdf.index[0]
dflink.loc[ind, 'ev2'] = tempdf.columns[0]
lags = _traceEventDendro(dflink, cx, lags, CCtoLag, clustDict,
clustDict.iloc[-1])
return lags, dflink
# return clusts,lagByClust
def _traceEventDendro(dflink, x, lags, CCtoLag, clustDict, clus):
"""
Function to follow ind1 through clustering linkage in linkup
and return total offset time (used for alignment)
"""
if len(dflink) < 1: # if event is singleton return zero offset time
lagSeries = pd.Series([0], index=clus)
else:
allevents = dflink['II'][dflink.index.values.max()]
allevents.sort()
# total lags for each station
lagSeries = pd.Series([0] * (len(dflink) + 1), index=allevents)
for a in dflink.iterrows():
if a[1].ev1 in clustDict[int(a[1].i1)]:
cl22 = clustDict[int(a[1].i2)]
else:
cl22 = clustDict[int(a[1].i1)]
# reference cc to lag samps map, round and cast to sample int
currentLag = int(np.round(CCtoLag[a[1].cc]))
for b in cl22: # record and update lags for second cluster
lagSeries[b] += currentLag
lags = _updateLags(b, lags, len(dflink), currentLag)
CCtoLag = _makeCC2LagMap(x, lags)
return lagSeries
def _updateLags(evenum, lags, N, currentLag):
"""
function to add current lag shifts to effected
lag times (see Haris 2006 appendix B)
"""
dow = _getDow(N, evenum) # get the index to add to lags for columns
acr = _getAcr(N, evenum)
for a in acr:
lags[a] += currentLag
for a in dow:
lags[a] -= currentLag
return lags
def _getDow(N, evenum):
dow = [0] * evenum
if len(dow) > 0:
for a in range(len(dow)):
dow[a] = _triangular(N - 1) - 1 + evenum - _triangular(N - 1 - a)
return dow
def _getAcr(N, evenum):
acr = [0] * (N - evenum)
if len(acr) > 0:
acr[0] = _triangular(N) - _triangular(N - (evenum))
for a in range(1, len(acr)):
acr[a] = acr[a - 1] + 1
return acr
def _triangular(n):
"""
calculate sum of triangle with base N,
see http://en.wikipedia.org/wiki/Triangular_number
"""
return sum(range(n + 1))
def _getClustDict(linkup, N):
"""
get pd series that will define the base events in each cluster
(including intermediate clusters)
"""
inds = np.arange(0, N + 1)
content = [np.array([x]) for x in np.arange(0, N + 1)]
clusdict = pd.Series(content, index=inds)
for a in range(len(linkup)):
ind1 = int(linkup[a, 4])
ind2 = int(linkup[a, 0])
ind3 = int(linkup[a, 1])
clusdict[ind1] = np.append(clusdict[ind2], clusdict[ind3])
return clusdict
def _ensureUnique(cx, cxdf):
"""
Make sure each coeficient is unique so it can be used as a key to
reference time lags, if not unique perturb slightly
"""
se = pd.Series(cx)
dups = se[se.duplicated()]
count = 0
while len(dups) > 0:
msg = ('Duplicates found in correlation coefficients,'
'perturbing slightly to get unique values')
detex.log(__name__, msg, level='warning', pri=True)
for a in dups.iteritems():
se[a[0]] = a[1] - abs(.00001 * np.random.rand())
count += 1
dups = se[se.duplicated()]
if count > 10:
msg = 'cannot make Coeficients unique, killing program'
detex.log(__name__, msg, level='error')
if count > 1: # if the cx has been perturbed update cxdf
for a in range(len(cxdf)):
sindex = sum(pd.isnull(a[1]))
tri1 = _triangular(len(cxdf))
tri2 = _triangular(len(cxdf) - a)
tri3 = _triangular(len(cxdf) - (a + 1))
cxdf.values[a, sindex:] = cx[tri1 - tri2, tri1 - tri3]
return se.values, cxdf
def _makeCC2LagMap(x, lags):
LS = pd.Series(lags, index=x)
return LS
def _loadStream(fetcher, filt, trim, decimate, station, dtype,
temkey, stakey, enforceOrigin=False, phases=None):
"""
loads all traces into stream object and applies filters and trims
"""
StreamDict = {} # Initialize dictionary for stream objects
channelDict = {}
stats = {}
STlens = {}
trLen = [] # trace length
allzeros = [] # empty list to stuff all zero keys to remove later
csta = stakey[stakey.STATION == station.split('.')[1]]
# load waveforms
for st, evename in fetcher.getTemData(temkey, csta, trim[0], trim[1],
returnName=True, phases=phases):
st = _applyFilter(st, filt, decimate, dtype)
if st is None or len(st) < 1:
continue # skip if stream empty
tem = temkey[temkey.NAME == evename]
if len(tem) < 1: # in theory this should never happen
msg = '%s not in template key, skipping'
detex.log(__name__, msg, pri=True)
continue
originTime = obspy.UTCDateTime(tem.iloc[0].TIME)
Nc = len(set([x.stats.channel for x in st])) # get number of channels
if Nc != len(st) or len(st) == 0:
msg = ('%s on %s is fractured or channels are missing, consider '
'setting fillZeros to True in ClusterStream to try to '
'make it usable, skipping') % (evename, station)
detex.log(__name__, msg, pri=True)
continue
if enforceOrigin: # if the waveforms should start at the origin
st.trim(starttime=originTime, pad=True, fill_value=0.0)
StreamDict[evename] = st
channelDict[evename] = [x.stats.channel for x in st]
pros = st[0].stats['processing']
sr = st[0].stats.sampling_rate
start = st[0].stats.starttime.timestamp
statsdi = {'processing': pros, 'sampling_rate': sr, 'starttime': start,
'Nc': Nc}
stats[evename] = statsdi # put stats in dict
totalLength = np.sum([len(x) for x in st])
if any([not np.any(x.data) for x in st]):
allzeros.append(evename)
trLen.append(totalLength)
STlens[evename] = totalLength
mlen = np.median(trLen)
keysToRemove = [x for x in StreamDict.keys() if STlens[x] < mlen * .2]
for key in keysToRemove: # Remove fractured or very short waveforms
msg = '%s is fractured or missing data, removing' % key
detex.log(__name__, msg, level='warning', pri=True)
StreamDict.pop(key, None)
channelDict.pop(key, None)
stats.pop(key, None)
for key in set(allzeros): # remove waveforms with channels filled with 0s
msg = '%s has at least one channel that is all zeros, deleting' % key
detex.log(__name__, msg, level='warning', pri=True)
StreamDict.pop(key, None)
channelDict.pop(key, None)
stats.pop(key, None)
if len(StreamDict.keys()) < 2:
msg = ('Less than 2 events survived preprocessing for station'
'%s Check input parameters, especially trim' % station)
detex.log(__name__, msg, level='warning', pri=True)
return None, None, None, None
evlist = StreamDict.keys()
evlist.sort()
# if 'IMU' in station:
return StreamDict, evlist, channelDict, stats
[docs]def multiplex(st, Nc=None, trimTolerance=15, template=False, returnlist=False,
retst=False):
"""
Multiplex an obspy stream object
Parameters
----------
st : instance of obspy stream
The stream containing the data to multiplex.
Nc : None or int
if not None the number of channels in stream, else try to determine
trimTolerance : int
The number of samples each channel can vary before being rejected
Template : bool
If True st is a template waveform, therefore an exception will be
raised if trimeTolerance is exceeded
returnlist : bool
If true also return np array of un-multiplexed data as a list
Returns
------
list with multiplexed data and other desired waveforms
"""
if Nc is None:
Nc = len(set([x.stats.station for x in st]))
if Nc == 1: # If only one channel do nothing
C1 = st[0].data
C = st[0].data
else:
chans = [x.data for x in st] # Data on each channel
minlen = np.array([len(x) for x in chans])
if max(minlen) - min(minlen) > trimTolerance:
netsta = st[0].stats.network + '.' + st[0].stats.station
utc1 = str(st[0].stats.starttime).split('.')[0]
utc2 = str(st[0].stats.endtime).split('.')[0]
msg = ('Channel lengths are not within %d on %s from %s to %s' %
(trimTolerance, netsta, utc1, utc2))
if template:
detex.log(__name__, msg, level='error')
else:
msg = msg + ' trimming to shortest channel'
detex.log(__name__, msg, level='warning', pri=True)
trimDim = min(minlen) # trim to smalles dimension
chansTrimed = [x[:trimDim] for x in chans]
elif max(minlen) - min(minlen) > 0: # if all channels not equal lengths
trimDim = min(minlen)
chansTrimed = [x[:trimDim] for x in chans] # trim to shortest
elif max(minlen) - min(minlen) == 0: # if chan lengths are exactly equal
chansTrimed = chans
C = np.vstack((chansTrimed))
C1 = np.ndarray.flatten(C, order='F')
out = [C1] # init output list
if returnlist:
out.append(C)
if retst:
out.append(st)
if len(out) == 1:
return out[0]
else:
return out
def _applyFilter(st, filt, decimate=False, dtype='double', fillZeros=False):
"""
Apply a filter, decimate, and trim to even start/end times
"""
if st is None or len(st) < 1:
msg = '_applyFilter got a stream with 0 length'
detex.log(__name__, msg, level='warn')
return obspy.Stream()
st.sort()
st1 = st.copy()
if dtype == 'single': # cast into single
for num, tr in enumerate(st):
st[num].data = tr.data.astype(np.float32)
nc = list(set([x.stats.channel for x in st]))
if len(st) > len(nc): # if data is fragmented only keep largest chunk
if fillZeros:
st = _mergeChannelsFill(st)
else:
st = _mergeChannels(st)
if not len(st) == len(nc) or len(st) < 1:
sta = st1[0].stats.station
stime = str(st1[0].stats.starttime)
msg = 'Stream is too fractured around %s on %s' % (str(stime), sta)
detex.log(__name__, msg, level='warn')
return obspy.Stream()
# st1.write('failed_merge-%s-%s.pkl'%(sta, stime), 'pickle')
# assert len(st) == len(nc)
if decimate:
st.decimate(decimate)
startTrim = max([x.stats.starttime for x in st])
endTrim = min([x.stats.endtime for x in st])
if startTrim > endTrim: # return empty string if chans dont overlap
return obspy.Stream()
st.trim(starttime=startTrim, endtime=endTrim)
st = st.split()
st.detrend('linear')
if isinstance(filt, list) or isinstance(filt, tuple):
st.filter('bandpass', freqmin=filt[0], freqmax=filt[1],
corners=filt[2], zerophase=filt[3])
return st
def _mergeChannels(st):
"""
function to find longest continuous data chunck and discard the rest
"""
st1 = st.copy()
st1.merge(fill_value=0.0)
start = max([x.stats.starttime for x in st1])
end = min([x.stats.endtime for x in st1])
try:
st1.trim(starttime=start, endtime=end)
except ValueError: # if stream too factured end is larger than start
return obspy.Stream()
ar_len = min([len(x.data) for x in st1])
ar = np.ones(ar_len)
for tr in st1:
ar *= tr.data
trace = obspy.Trace(data=np.ma.masked_where(ar == 0.0, ar))
trace.stats.starttime = start
trace.stats.sampling_rate = st1[0].stats.sampling_rate
if (ar == 0.0).any():
try:
st2 = trace.split()
except Exception:
return obspy.Stream()
times = np.array([[x.stats.starttime, x.stats.endtime] for x in st2])
df = pd.DataFrame(times, columns=['start', 'stop'])
df['duration'] = df['stop'] - df['start']
max_dur = df[df.duration == df['duration'].max()].iloc[0]
st.trim(starttime=max_dur.start, endtime=max_dur.stop)
else:
st = st1
return st
def _mergeChannelsFill(st):
st.merge(fill_value=0.0)
return st
def _checkClusterInputs(filt, dtype, trim, decimate):
"""
Check a few key input parameters to make sure everything is kosher
"""
if filt is not None and len(filt) != 4: # check filt
msg = 'filt must either be None (no filter) or a len 4 list or tuple'
detex.log(__name__, msg, level='error')
if dtype != 'double' and dtype != 'single': # check dtype
msg = ('dype must be either "double" or "single" not %s, setting to \
double' % dtype)
dtype = 'double'
detex.log(__name__, msg, level='warn', pri=True)
if trim is not None: # check trim
if len(trim) != 2:
msg = 'Trim must be a list or tuple of length 2'
detex.log(__name__, msg, level='warn', pri=True)
else:
if -trim[0] > trim[1]:
msg = 'Invalid trim parameters'
detex.log(__name__, msg, level='error')
if decimate is not None:
if not isinstance(decimate, int):
msg = 'decimate must be an int'
detex.log(__name__, msg, level='error', e=TypeError)