# -*- coding: utf-8 -*-
"""
Created on Tue Jul 08 21:24:18 2014
@author: Derrick
Module containing import detex classes
"""
# python 2 and 3 compatibility imports
from __future__ import print_function, absolute_import, unicode_literals, division
import json
import numbers
import os
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import obspy
import pandas as pd
import scipy
from six import string_types
import detex
try: # python 2/3 compat
import cPickle
except ImportError:
import pickle as cPickle
import itertools
import copy
import colorsys
from struct import pack
import PyQt4
import sys
from scipy.cluster.hierarchy import dendrogram, fcluster
from detex.detect import _SSDetex
pd.options.mode.chained_assignment = None # mute setting copy warning
# warnings.filterwarnings('error') #uncomment this to make all warnings errors
# lines for backward compat.
[docs]class ClusterStream(object):
"""
A container for multiple cluster objects, should only be called with
detex.construct.createCluster
"""
def __init__(self, trdf, temkey, stakey, fetcher, eventList, ccReq, filt,
decimate, trim, fileName, eventsOnAllStations, enforceOrigin):
self.__dict__.update(locals()) # Instantiate all input variables
self.ccReq = None # set to None because it can vary between stations
self.clusters = [0] * len(trdf)
self.stalist = trdf.Station.values.tolist() # station lists
self.stalist2 = [x.split('.')[1] for x in self.stalist]
self.filename = fileName
self.eventCodes = self._makeCodes()
for num, row in trdf.iterrows():
if not eventsOnAllStations:
evlist = row.Events
else:
evlist = eventList
self.clusters[num] = Cluster(self, row.Station, temkey, evlist,
row.Link, ccReq, filt, decimate, trim,
row.CCs)
def _makeObsLine(self, sta, dt, cc, pha='S'):
line = '%s %0.4f %0.4f %s' % (sta, dt, cc, pha)
return line
def _makeHeader(self, num1, num2, reqZeros):
fomatstr = '{:0' + "{:d}".format(reqZeros) + 'd}'
# assume cross corr and cat origins are identical
head = '# ' + fomatstr.format(num1) + \
' ' + fomatstr.format(num2) + ' ' + '0.0'
return head
def _makeCodes(self):
evcodes = {}
for num, row in self.temkey.iterrows():
evcodes[num] = row.NAME
return evcodes
[docs] def updateReqCC(self, reqCC):
"""
Updates the required correlation coefficient for clusters to form on
all stations or individual stations.
Parameters
--------------
reqCC : float (between 0 and 1), or dict of reference keys and floats
if reqCC is a float the required correlation coeficient for
clusters to form will be set to reqCC on all stations.
If dict keys must be indicies for each cluster object (IE net.sta,
sta, or int index) and values are the reqCC for that station.
Notes
---------------
The Cluster class also have a similar method that can be more
intuitive to use, as in the tutorial
"""
if isinstance(reqCC, float):
if reqCC < 0 or reqCC > 1:
msg = 'reqCC must be between 0 and 1'
detex.log(__name__, msg, level='error')
for cl in self.clusters:
cl.updateReqCC(reqCC)
elif isinstance(reqCC, dict):
for key in reqCC.keys():
self[key].updateReqCC(reqCC[key])
elif isinstance(reqCC, list):
for num, cc in enumerate(reqCC):
self[num].updateReqCC(cc)
[docs] def printAtr(self): # print out basic attributes used to make cluster
for cl in self.clusters:
cl.printAtr()
[docs] def dendro(self, **kwargs):
"""
Create dendrograms for each station
"""
for cl in self.clusters:
cl.dendro(**kwargs)
[docs] def simMatrix(self, groupClusts=False, savename=False, returnMat=False,
**kwargs):
"""
Function to create similarity matrix of each event pair
Parameters
-------
groupClusts : bool
If True order by clusters on the simmatrix with the singletons
coming last
savename : str or False
If not False, a path used by plt.savefig to save the current
figure. The extension is necesary for specifying format.
See plt.savefig for details
returnMat : bool
If true return the similarity matrix
"""
out = []
for cl in self.clusters:
dout = cl.simMatrix(groupClusts, savename, returnMat, **kwargs)
out.append(dout)
[docs] def plotEvents(self, projection='merc', plotSingles=True, **kwargs):
"""
Plot the event locations for each station using basemap. Calls the
plotEvents method of the Cluster class, see its docs for accepted
kwargs.
Parameters
---------
projection : str
The pojection type to pass to basemap
plotSingles : bool
If True also plot the singletons (events that dont cluster)
Notes
-------
kwargs are passed to basemap
If no working installation of basemap is found an ImportError will
be raised. See the following URL for tips on installing it:
http://matplotlib.org/basemap/users/installing.html, good luck!
"""
for cl in self.clusters:
cl.plotEvents(projection, plotSingles, **kwargs)
[docs] def write(self): # uses pickle to write class to disk
"""
Write instance to file (name is the filename attribute)
"""
msg = 'writing ClusterStream instance as %s' % self.filename
detex.log(__name__, msg, level='info', pri=True)
cPickle.dump(self, open(self.filename, 'wb'))
def __getitem__(self, key): # allows indexing of children Cluster objects
if isinstance(key, int):
return self.clusters[key]
elif isinstance(key, string_types):
if len(key.split('.')) == 1:
return self.clusters[self.stalist2.index(key)]
elif len(key.split('.')) == 2:
return self.clusters[self.stalist.index(key)]
else:
msg = ('indexer must either be an int or str of sta.net or sta'
' you passed %s' % key)
detex.log(__name__, msg, level='error')
def __len__(self):
return len(self.clusters)
def __repr__(self):
outstr = 'SSClusterStream with %d stations ' % (len(self.stalist))
return outstr
[docs]class Cluster(object):
def __init__(self, clustStream, station, temkey, eventList, link, ccReq,
filt, decimate, trim, DFcc):
# instantiate a few needed varaibles (not all to save space)
self.link = link
self.DFcc = DFcc
self.station = station
self.temkey = temkey
self.key = eventList
self.updateReqCC(ccReq)
self.trim = trim
self.decimate = decimate
self.nonClustColor = '0.6' # use a grey of 0.6 for singletons
[docs] def updateReqCC(self, newccReq):
"""
Function to update the required correlation coeficient for
this station
Parameters
-------------
newccReq : float (between 0 and 1)
Required correlation coef
"""
if newccReq < 0. or newccReq > 1.:
msg = 'Parameter ccReq must be between 0 and 1'
detex.log(__name__, msg, level='error')
self.ccReq = newccReq
self.dflink, serclus = self._makeDFLINK(truncate=False)
# get events that actually cluster (filter out singletons)
dfcl = self.dflink[self.dflink.disSim <= 1 - self.ccReq]
# sort putting highest links in cluster on top
dfcl.sort_values(by='disSim', inplace=True, ascending=False)
dfcl.reset_index(inplace=True, drop=True)
dftemp = dfcl.copy()
clustlinks = {}
clustEvents = {}
clnum = 0
while len(dftemp) > 0:
ser = dftemp.iloc[0]
ndf = dftemp[[set(x).issubset(ser.II) for x in dftemp.II]]
clustlinks[clnum] = ndf.clust
valset = set([y for x in ndf.II.values for y in x])
clustEvents[clnum] = list(valset)
dftemp = dftemp[~dftemp.index.isin(ndf.index)]
clnum += 1
self.clustlinks = clustlinks
self.clusts = [[self.key[y] for y in clustEvents[x]]
for x in clustEvents.keys()]
keyset = set(self.key)
clustset = set([y for x in self.clusts for y in x])
self.singles = list(keyset.difference(clustset))
self.clustcount = np.sum([len(x) for x in self.clusts])
self.clustColors = self._getColors(len(self.clusts))
msg = ('ccReq for station %s updated to ccReq=%1.3f' %
(self.station, newccReq))
detex.log(__name__, msg, level='info', pri=True)
def _getColors(self, numClusts):
"""
See if there are enough defualt colors for the clusters, if not
Generate N unique colors (that probably dont look good together)
"""
clustColorsDefault = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
# if there are enough default python colors use them
if numClusts <= len(clustColorsDefault):
return clustColorsDefault[:numClusts]
else: # if not generaete N unique colors
colors = []
for i in np.arange(0., 360., 360. / numClusts):
hue = i / 360.
lightness = (50 + np.random.rand() * 10) / 100.
saturation = (90 + np.random.rand() * 10) / 100.
cvect = colorsys.hls_to_rgb(hue, lightness, saturation)
rgb = [int(x * 255) for x in cvect]
# covnert to hex code
colors.append('#' + pack("BBB", *rgb).encode('hex'))
return colors
def _makeColorDict(self, clustColors, nonClustColor):
if len(self.clusts) < 1:
colorsequence = clustColors
# if not enough colors repeat color matrix
elif float(len(clustColors)) / len(self.clusts) < 1:
colorsequence = clustColors * \
int(np.ceil((float(len(self.clusts)) / len(clustColors))))
else:
colorsequence = clustColors
# unitialize color list with default color
color_list = [nonClustColor] * 3 * len(self.dflink)
for a in range(len(self.clusts)):
for b in self.clustlinks[a]:
color_list[int(b)] = colorsequence[a]
return color_list
def _makeDFLINK(self, truncate=True): # make the link dataframe
N = len(self.link)
# append cluster numbers to link array
link = np.append(self.link, np.arange(N + 1, N + N + 1).reshape(N, 1), 1)
if truncate: # truncate after required coeficient
linkup = link[link[:, 2] <= 1 - self.ccReq]
else:
linkup = link
T = fcluster(link[:, 0:4], 1 - self.ccReq, criterion='distance')
serclus = pd.Series(T)
clusdict = pd.Series([np.array([x]) for x in np.arange(
0, N + 1)], index=np.arange(0, N + 1))
for a in range(len(linkup)):
clusdict[int(linkup[a, 4])] = np.append(
clusdict[int(linkup[a, 0])], clusdict[int(linkup[a, 1])])
columns = ['i1', 'i2', 'disSim', 'num', 'clust']
dflink = pd.DataFrame(linkup, columns=columns)
if len(dflink) > 0:
dflink['II'] = list
else:
msg = 'No events cluster with corr coef = %1.3f' % self.ccReq
detex.log(__name__, msg, level='info', pri=True)
for a in dflink.iterrows(): # enumerate cluster contents
ar1 = list(np.array(clusdict[int(a[1].i1)]))
ar2 = list(np.array(clusdict[int(a[1].i2)]))
dflink['II'][a[0]] = ar1 + ar2
return dflink, serclus
# creates a basic dendrogram plot
[docs] def dendro(self, hideEventLabels=True, show=True, saveName=False,
legend=True, **kwargs):
"""
Function to plot dendrograms of the clusters
Parameters
-----
hideEventLabels : bool
turns x axis labeling on/off. Better set to false
if many events are in event pool
show : bool
If true call plt.show
saveName : str or False
path to save figure. Extention denotes format. See plt.savefig
for details
legend : bool
If true plot a legend on the side of the dendrogram
Note
----------
kwargs are passed to scipy.cluster.hierarchy.dendrogram, see docs
for acceptable arguments and descriptions
"""
# Get color schemes
color_list = self._makeColorDict(self.clustColors, self.nonClustColor)
for a in range(len(self.clusts)):
plt.plot([], [], '-', color=self.clustColors[a])
plt.plot([], [], '-', color=self.nonClustColor)
dendrogram(self.link, color_threshold=1 - self.ccReq, count_sort=True,
link_color_func=lambda x: color_list[x], **kwargs)
ax = plt.gca()
if legend:
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend([str(x) for x in range(1, len(self.clusts) + 1)] +
['N/A'], loc='center left', bbox_to_anchor=(1, .5),
title='Clusters')
ax.set_ylim([0, 1])
if hideEventLabels:
ax.set_xticks([])
plt.xlabel('Events')
plt.ylabel('Dissimilarity')
plt.title(self.station)
if saveName:
plt.savefig(saveName, **kwargs)
if show:
plt.show()
[docs] def plotEvents(self, projection='merc', plotSingles=True, **kwargs):
"""
Plot the event locations for each station using basemap. Calls the
plotEvents method of the Cluster class, see its docs for accepted
kwargs.
Parameters
---------
projection : str
The pojection type to pass to basemap
plotSingles : bool
If True also plot the singletons (events that dont cluster)
Notes
-------
kwargs are passed to basemap
If no working installation of basemap is found an ImportError will
be raised. See the following URL for tips on installing it:
http://matplotlib.org/basemap/users/installing.html, good luck!
"""
# TODO make dot size scale with magnitudes
# make sure basemap is installed
try:
from mpl_toolkits.basemap import Basemap
except ImportError:
msg = 'mpl_toolskits basemap not installed, cant plot'
detex.log(__name__, msg, level='error', e=ImportError)
# init figures and get limits
fig_map, emap, horrange = self._init_map(Basemap, projection, kwargs)
zmin, zmax, zscale = self._get_z_scaling(horrange)
fig_lat = self._init_profile_figs(zmin, zmax, zscale)
fig_lon = self._init_profile_figs(zmin, zmax, zscale)
# seperate singletons from clustered events
cl_dfs, sing_df = self._get_singletons_and_clusters()
self._plot_map_view(emap, fig_map, horrange, cl_dfs, sing_df)
self._plot_profile_view(zmin, zmax, zscale, fig_lat, fig_lon, cl_dfs,
sing_df, emap)
def _init_map(self, Basemap, projection, kwargs):
"""
Function to setup the map figure with basemap returns the
figure instance and basemap instance and horizontal range of plot
"""
map_fig = plt.figure()
# get map bounds
latmin = self.temkey.LAT.min()
latmax = self.temkey.LAT.max()
lonmin = self.temkey.LON.min()
lonmax = self.temkey.LON.max()
# create buffers so there is a slight border with no events around map
latbuff = abs((latmax - latmin) * 0.1)
lonbuff = abs((lonmax - lonmin) * 0.1)
# get the total horizontal distance of plot in km
totalxdist = obspy.core.util.geodetics.gps2DistAzimuth(
latmin, lonmin, latmin, lonmax)[0] / 1000
# init projection
emap = Basemap(projection=projection,
lat_0=np.mean([latmin, latmax]),
lon_0=np.mean([lonmin, lonmax]),
resolution='h',
area_thresh=0.1,
llcrnrlon=lonmin - lonbuff,
llcrnrlat=latmin - latbuff,
urcrnrlon=lonmax + lonbuff,
urcrnrlat=latmax + latbuff,
**kwargs)
# draw scale
emap.drawmapscale(lonmin, latmin, lonmin, latmin, totalxdist / 4.5)
# get limits in projection
xmax, xmin, ymax, ymin = emap.xmax, emap.xmin, emap.ymax, emap.ymin
horrange = max((xmax - xmin), (ymax - ymin)) # horizontal range
# get maximum degree distance for setting scalable ticks
latdi, londi = [abs(latmax - latmin), abs(lonmax - lonmin)]
maxdeg = max(latdi, londi)
parallels = np.arange(0., 80, maxdeg / 4)
emap.drawparallels(parallels, labels=[1, 0, 0, 1])
meridians = np.arange(10., 360., maxdeg / 4)
mers = emap.drawmeridians(meridians, labels=[1, 0, 0, 1])
for m in mers: # rotate meridian labels
try:
mers[m][1][0].set_rotation(90)
except:
pass
plt.title('Clusters on %s' % self.station)
return map_fig, emap, horrange
def _init_profile_figs(self, zmin, zmax, zscale):
"""
init figs for plotting the profiles of the events
"""
# init profile figures
profile_fig = plt.figure()
z1 = zmin * zscale
z2 = zmax * zscale
tickfor = ['%0.1f' % x1 for x1 in np.linspace(zmin, zmax, 10)]
plt.yticks(np.linspace(z1, z2, 10), tickfor)
plt.gca().invert_yaxis()
plt.xticks([])
plt.ylabel('Depth (km)')
return profile_fig
def _get_z_scaling(self, horrange):
"""
Return z limits and scale factors
"""
zmin, zmax = self.temkey.DEPTH.min(), self.temkey.DEPTH.max()
zscale = horrange / (zmax - zmin)
return zmin, zmax, zscale
def _get_singletons_and_clusters(self):
"""
get dataframes of clustered events and singletons
Note: cl_dfs is a list of dfs whereas sing_df is just a df
"""
cl_dfs = [self.temkey[self.temkey.NAME.isin(x)] for x in self.clusts]
sing_df = self.temkey[self.temkey.NAME.isin([x for x in self.singles])]
return cl_dfs, sing_df
def _plot_map_view(self, emap, map_fig, horrange, cl_dfs, sing_df):
"""
plot the map figure
"""
plt.figure(map_fig.number) # set to map figure
# plot singles
x, y = emap(sing_df.LON.values, sing_df.LAT.values)
emap.plot(x, y, '.', color=self.nonClustColor, ms=6.0)
for clnum, cl in enumerate(cl_dfs):
x, y = emap(cl.LON.values, cl.LAT.values)
emap.plot(x, y, '.', color=self.clustColors[clnum])
def _plot_profile_view(self, zmin, zmax, zscale, fig_lat, fig_lon, cl_df,
sing_df, emap):
"""
plot the profile view
"""
x_sing, y_sing = emap(sing_df.LON.values, sing_df.LAT.values)
# plot singletons
nccolor = self.nonClustColor
plt.figure(fig_lon.number)
plt.plot(x_sing, sing_df.DEPTH * zscale, '.', color=nccolor, ms=6.0)
plt.xlabel('Longitude')
plt.figure(fig_lat.number)
plt.plot(y_sing, sing_df.DEPTH * zscale, '.', color=nccolor, ms=6.0)
plt.xlabel('Latitude')
# plot clusters
for clnum, cl in enumerate(cl_df):
ccolor = self.clustColors[clnum]
x, y = emap(cl.LON.values, cl.LAT.values)
plt.figure(fig_lon.number)
plt.plot(x, cl.DEPTH * zscale, '.', color=ccolor)
plt.figure(fig_lat.number)
plt.plot(y, cl.DEPTH * zscale, '.', color=ccolor)
# set buffers so nothing plots right on edge
for fig in [fig_lat, fig_lon]:
plt.figure(fig.number)
xlim = plt.xlim()
xdist = abs(max(xlim) - min(xlim))
plt.xlim(xlim[0] - xdist * .1, xlim[1] + xdist * .1)
ylim = plt.ylim()
ydist = abs(max(xlim) - min(xlim))
plt.ylim(ylim[0] - ydist * .1, ylim[1] + ydist * .1)
[docs] def simMatrix(self, groupClusts=False, savename=False, returnMat=False,
**kwargs):
"""
Function to create basic similarity matrix of the values
in the cluster object
Parameters
-------
groupClusts : boolean
If True order by clusters on the simmatrix with the
singletons coming last
savename : str or False
If not False, a path used by plt.savefig to save the current
figure. The extension is necesary for specifying format. See
plt.savefig for details
returnMat : boolean
If true return the similarity matrix
"""
if groupClusts: # if grouping clusters together
clusts = copy.deepcopy(self.clusts) # get cluster list
clusts.append(self.singles) # add singles list at end
eveOrder = list(itertools.chain.from_iterable(clusts))
indmask = {
num: list(self.key).index(eve) for num,
eve in enumerate(eveOrder)} # create a mask forthe order
else:
# blank index mask if not
indmask = {x: x for x in range(len(self.key))}
plt.figure()
le = self.DFcc.columns.values.max()
mat = np.zeros((le + 1, le + 1))
# deb([le,indmask,self.DFcc])
for a in range(le + 1):
for b in range(le + 1):
if a == b:
mat[a, b] = 1
else:
# new a and b coords based on mask
a1, b1 = indmask[a], indmask[b]
gi = max(a1, b1)
li = min(a1, b1)
mat[a, b] = self.DFcc.loc[li, gi]
mat[b, a] = self.DFcc.loc[li, gi]
cmap = mpl.colors.LinearSegmentedColormap.from_list(
'my_colormap', ['blue', 'red'], 256)
img = plt.imshow(
mat,
interpolation='nearest',
cmap=cmap,
origin='upper',
vmin=0,
vmax=1)
plt.clim(0, 1)
plt.grid(True, color='white')
plt.colorbar(img, cmap=cmap)
plt.title(self.station)
if savename:
plt.savefig(savename, **kwargs)
if returnMat:
return mat
[docs] def write(self): # uses pickle to write class to disk
cPickle.dump(self, open(self.filename, 'wb'))
[docs] def printAtr(self): # print out basic attributes used to make cluster
print('%s Cluster' % self.station)
print('%d Events cluster out of %d' %
(self.clustcount, len(self.singles) + self.clustcount))
print('Total number of clusters = %d' % len(self.clusts))
print('Required Cross Correlation Coeficient = %.3f' % self.ccReq)
def __getitem__(self, index): # allow indexing
return self.clusts[index]
def __iter__(self): # make class iterable
return iter(self.clusts)
def __len__(self):
return len(self.clusts)
# def __repr__(self):
# self.printAtr()
# return ''
[docs]class SubSpace(object):
""" Class used to hold subspaces for detector
Holds both subspaces (as defined from the SScluster object) and
single event clusters, or singles
"""
def __init__(self, singlesDict, subSpaceDict, cl, dtype, Pf, cfetcher):
self.cfetcher = cfetcher
self.clusters = cl
self.subspaces = subSpaceDict
self.singles = singlesDict
self.singletons = singlesDict
self.dtype = dtype
self.Pf = Pf
self.ssStations = self.subspaces.keys()
self.singStations = self.singles.keys()
self.Stations = list(set(self.ssStations) | set(self.singStations))
self.Stations.sort()
self._stakey2 = {x: x for x in self.ssStations}
self._stakey1 = {x.split('.')[1]: x for x in self.ssStations}
################################ Validate Cluster functions
[docs] def validateClusters(self):
"""
Method to check for misaligned waveforms and discard those that no
longer meet the required correlation coeficient for each cluster.
See Issue 25 (www.github.com/d-chambers/detex) for why this might
be useful.
"""
msg = 'Validating aligned (and trimmed) waveforms in each cluster'
detex.log(__name__, msg, level='info', pri=True)
for sta in self.subspaces.keys():
subs = self.subspaces[sta]
c = self.clusters[sta]
ccreq = c.ccReq
for clustNum, row in subs.iterrows():
stKeys = row.SampleTrims.keys()
# get trim times if defined
if 'Starttime' in stKeys and 'Endtime' in stKeys:
start = row.SampleTrims['Starttime']
stop = row.SampleTrims['Endtime']
else:
start = 0
stop = -1
for ev1num, ev1 in enumerate(row.Events[:-1]):
ccs = [] # blank list for storing ccs of aligned WFs
for ev2 in row.Events[ev1num + 1:]:
t = row.AlignedTD[ev1][start: stop]
s = row.AlignedTD[ev2][start: stop]
maxcc = detex.construct.fast_normcorr(t, s)
ccs.append(maxcc)
if len(ccs) > 0 and max(ccs) < ccreq:
msg = (('%s fails validation check or is ill-aligned '
'on station %s, removing') % (ev1, row.Station))
detex.log(__name__, msg, pri=True)
self._removeEvent(sta, ev1, clustNum)
msg = 'Finished validateCluster call'
detex.log(__name__, msg, level='info', pri=True)
def _removeEvent(self, sta, event, clustNum):
"""
Function to remove an event from a SubSpace instance
"""
# remove from eventList
srow = self.subspaces[sta].loc[clustNum]
srow.Events.remove(event)
srow.AlignedTD.pop(event, None)
################################ SVD Functions
[docs] def SVD(self, selectCriteria=2, selectValue=0.9, conDatNum=100,
threshold=None, normalize=False, useSingles=True,
validateWaveforms=True, backupThreshold=None, **kwargs):
"""
Function to perform SVD on the alligned waveforms and select which
of the SVD basis are to be used in event detection. Also assigns
a detection threshold to each subspace-station pair.
Parameters
----------------
selctionCriteria : int, selectValue : number
selectCriteria is the method for selecting which basis vectors
will be used as detectors. selectValue depends on selectCriteria
Valid options are:
0 - using the given Pf, find number of dimensions to maximize
detection probability !!! NOT YET IMPLIMENTED!!!
selectValue - Not used
(Need to find a way to use the doubly-non central F
distribution in python)
1 - Failed implementation, not supported
2 - select basis number based on an average fractional signal
energy captured (see Figure 8 of Harris 2006). Then calculate
an empirical distribution of the detection statistic by running
each subspace over random continuous data with no high amplitude
signals (see getFAS method). A beta distribution is then fit to
the data and the DS value that sets the probability of false
detection to the Pf defined in the subspace instance is selected
as the threshold.
selectValue - Average fractional energy captured,
can range from 0 (use no basis vectors) to 1
(use all basis vectors). A value between 0.75 and 0.95
is recommended.
3 - select basis number based on an average fractional signal
energy captured (see Figure 8 of Harris 2006).
Then set detection threshold to a percentage of the minimum
fractional energy captured. This method is a bit quick and dirty
but ensures all events in the waveform pool will be detected.
select value is a fraction representing the fraction of
the minum fractional energy captured (between 0 and 1).
4 - use a user defined number of basis vectors, beginning with the
most significant (Barrett and Beroza 2014 use first two basis
vectors as an "empirical" subspace detector). Then use the same
technique in method one to set threshold
selectValue - can range from 0 to number of events in
subspace, if selectValue is greater than number of events
all events are used
conDatNum : int
The number of continuous data chunks to use to estimate the
effective dimension of the signal space or to estimate the null
distribution. Used if selectCriteria == 1,2,4
threshold : float or None
Used to set each subspace at a user defined threshold. If any
value is set it overrides any of the previously defined methods and
avoids estimating the effective dimension of representation or
distribution of the null space. Can be useful if problems arise
in the false alarm statistic calculation
normalize : bool
If true normalize the amplitude of all the training events before
preforming the SVD. Keeps higher amplitude events from dominating
the SVD vectors but can over emphasize noise. Haris 2006 recomends
using normalization but the personal experience of the author has
found normalization can increase the detector's propensity to
return false detections.
useSingles : bool
If True also calculate the thresholds for singles
validateWaveforms : bool
If True call the validateClusters method before the performing SVD
to make sure each trimed aligned waveform still meets the
required correlation coeficient. Any waveforms that do not will
be discarded.
backupThreshold : None or float
A backup threshold to use if approximation fails. Typically,
using the default detex settings, a reasonable value would be
0.25
kwargs are passed to the getFAS call (if used)
"""
# make sure user defined options are kosher
self._checkSelection(selectCriteria, selectValue, threshold)
# Iterate through all subspaces defined by stations
for station in self.ssStations:
for ind, row in self.subspaces[station].iterrows():
self.subspaces[station].UsedSVDKeys[ind] = []
svdDict = {} # initialize dict to put SVD vectors in
keys = sorted(row.Events)
arr, basisLength = self._trimGroups(ind, row, keys, station)
if basisLength == 0:
msg = (('subspace %d on %s is failing alignment and '
'trimming, deleting it') % (ind, station))
detex.log(__name__, msg, level='warn')
self._drop_subspace(station, ind)
continue
if normalize:
arr = np.array([x / np.linalg.norm(x) for x in arr])
tparr = np.transpose(arr)
# perform SVD
U, s, Vh = scipy.linalg.svd(tparr, full_matrices=False)
# make dict with sing. value as key and sing. vector as value
for einum, eival in enumerate(s):
svdDict[eival] = U[:, einum]
# asign Parameters back to subspace dataframes
self.subspaces[station].SVD[ind] = svdDict # assign SVD
fracEnergy = self._getFracEnergy(ind, row, svdDict, U)
usedBasis = self._getUsedBasis(ind, row, svdDict, fracEnergy,
selectCriteria, selectValue)
# Add fracEnergy and SVD keys (sing. vals) to main DataFrames
self.subspaces[station].FracEnergy[ind] = fracEnergy
self.subspaces[station].UsedSVDKeys[ind] = usedBasis
self.subspaces[station].SVDdefined[ind] = True
numBas = len(self.subspaces[station].UsedSVDKeys[ind])
self.subspaces[station].NumBasis[ind] = numBas
if len(self.ssStations) > 0:
self._setThresholds(selectCriteria, selectValue, conDatNum,
threshold, basisLength, backupThreshold, kwargs)
if len(self.singStations) > 0 and useSingles:
self.setSinglesThresholds(conDatNum=conDatNum, threshold=threshold,
backupThreshold=backupThreshold,
kwargs=kwargs)
def _drop_subspace(self, station, ssnum):
"""
Drop a subspace that is misbehaving
"""
space = self.subspaces[station]
self.subspaces[station] = space[space.index != int(ssnum)]
def _trimGroups(self, ind, row, keys, station):
"""
function to get trimed subspaces if trim times are defined, and
return an array of the aligned waveforms for the SVD to act on
"""
stkeys = row.SampleTrims.keys()
aliTD = row.AlignedTD
if 'Starttime' in stkeys and 'Endtime' in stkeys:
stim = row.SampleTrims['Starttime']
etim = row.SampleTrims['Endtime']
if stim < 0: # make sure stim is not less than 0
stim = 0
Arr = np.vstack([aliTD[x][stim:etim] -
np.mean(aliTD[x][stim:etim]) for x in keys])
basisLength = Arr.shape[1]
else:
msg = ('No trim times for %s and station %s, try running '
'pickTimes or attachPickTimes' % (row.Name, station))
detex.log(__name__, msg, level='warn', pri=True)
Arr = np.vstack([aliTD[x] - np.mean(aliTD[x]) for x in keys])
basisLength = Arr.shape[1]
return Arr, basisLength
def _checkSelection(self, selectCriteria, selectValue, threshold):
"""
Make sure all user defined values are kosher for SVD call
"""
if selectCriteria in [1, 2, 3]:
if selectValue > 1 or selectValue < 0:
msg = ('When selectCriteria==%d selectValue must be a float'
' between 0 and 1' % selectCriteria)
detex.log(__name__, msg, level='error', e=ValueError)
elif selectCriteria == 4:
if selectValue < 0 or not isinstance(selectValue, int):
msg = ('When selectCriteria==3 selectValue must be an'
'integer greater than 0')
detex.log(__name__, msg, level='error', e=ValueError)
else:
msg = 'selectCriteria of %s is not supported' % selectCriteria
detex.log(__name__, msg, level='error')
if threshold is not None:
if not isinstance(threshold, numbers.Number) or threshold < 0:
msg = 'Unsupported type for threshold, must be None or float'
detex.log(__name__, msg, level='error', e=ValueError)
def _getFracEnergy(self, ind, row, svdDict, U):
"""
calculates the % energy capture for each stubspace for each possible
dimension of rep. (up to # of events that go into the subspace)
"""
fracDict = {}
keys = row.Events
svales = svdDict.keys()
svales.sort(reverse=True)
stkeys = row.SampleTrims.keys() # dict defining sample trims
for key in keys:
aliTD = row.AlignedTD[key] # aligned waveform for event key
if 'Starttime' in stkeys and 'Endtime' in stkeys:
start = row.SampleTrims['Starttime'] # start of trim in samps
end = row.SampleTrims['Endtime'] # end of trim in samps
aliwf = aliTD[start: end]
else:
aliwf = aliTD
Ut = np.transpose(U) # transpose of basis vects
# normalized dot product (mat. mult.)
normUtAliwf = scipy.dot(Ut, aliwf) / scipy.linalg.norm(aliwf)
# add 0% energy capture for dim of 0
repvect = np.insert(np.square(normUtAliwf), 0, 0)
# cumul. energy captured for increasing dim. reps
cumrepvect = [np.sum(repvect[:x + 1]) for x in range(len(repvect))]
fracDict[key] = cumrepvect # add cumul. to keys
# get average and min energy capture, append value to dict
fracDict['Average'] = np.average([fracDict[x] for x in keys], axis=0)
fracDict['Minimum'] = np.min([fracDict[x] for x in keys], axis=0)
return (fracDict)
def _getUsedBasis(self, ind, row, svdDict, cumFracEnergy,
selectCriteria, selectValue):
"""
function to populate the keys of the selected SVD basis vectors
"""
keys = svdDict.keys()
keys.sort(reverse=True)
if selectCriteria in [1, 2, 3]:
# make sure last element is exactly 1
cumFracEnergy['Average'][-1] = 1.00
ndim = np.argmax(cumFracEnergy['Average'] >= selectValue)
selKeys = keys[:ndim] # selected keys
if selectCriteria == 4:
selKeys = keys[:selectValue + 1]
return selKeys
def _setThresholds(self, selectCriteria, selectValue, conDatNum,
threshold, basisLength, backupThreshold, kwargs={}):
if threshold > 0:
for station in self.ssStations:
subspa = self.subspaces[station]
for ind, row in subspa.iterrows():
self.subspaces[station].Threshold[ind] = threshold
elif selectCriteria == 1:
msg = 'selectCriteria 1 currently not supported'
detex.log(__name__, msg, level='error', e=ValueError)
elif selectCriteria in [2, 4]:
# call getFAS to estimate null space dist.
self.getFAS(conDatNum, **kwargs)
for station in self.ssStations:
subspa = self.subspaces[station]
for ind, row in subspa.iterrows():
beta_a, beta_b = row.FAS['betadist'][0:2]
# get threshold from beta dist.
# TODO consider implementing other dist. options as well
th = scipy.stats.beta.isf(self.Pf, beta_a, beta_b, 0, 1)
if th > .9:
th, Pftemp = self._approxThld(beta_a, beta_b, station,
row, self.Pf, 1000, 3,
backupThreshold)
msg = ('Scipy.stats.beta.isf failed with pf=%e, '
'approximated threshold to %f with a Pf of %e '
'for station %s %s using forward grid search' %
(self.Pf, th, Pftemp, station, row.Name))
detex.log(__name__, msg, level='warning')
self.subspaces[station].Threshold[ind] = th
elif selectCriteria == 3:
for station in self.ssStations:
subspa = self.subspaces[station]
for ind, row in subspa.iterrows():
th = row.FracEnergy['Minimum'][row.NumBasis] * selectValue
self.subspaces[station].Threshold[ind] = th
[docs] def setSinglesThresholds(self, conDatNum=50, recalc=False,
threshold=None, backupThreshold=None, **kwargs):
"""
Set thresholds for the singletons (unclustered events) by fitting
a beta distribution to estimation of null space
Parameters
----------
condatNum : int
The number of continuous data chunks to use to fit PDF
recalc : boolean
If true recalculate the the False Alarm Statistics
threshold : None or float between 0 and 1
If number, don't call getFAS simply use given threshold
backupThreshold : None or float
If approximate a threshold fails then use backupThreshold. If None
then raise.
Note
----------
Any singles without pick times will not be used. In this way singles
can be rejected
"""
for sta in self.singStations:
sing = self.singles[sta] # singles on station
sampTrims = self.singles[sta].SampleTrims
self.singles[sta].Name = ['SG%d' % x for x in range(len(sing))]
# get singles that have phase picks
singsAccepted = sing[[len(x.keys()) > 0 for x in sampTrims]]
self.singles[sta] = singsAccepted
self.singles[sta].reset_index(inplace=True, drop=True)
if threshold is None:
# get empirical dist unless manual threshold is passed
self.getFAS(conDatNum, useSingles=True,
useSubSpaces=False, **kwargs)
for sta in self.singStations:
for ind, row in self.singles[sta].iterrows():
if len(row.SampleTrims.keys()) < 1: # skip singles with no pick times
continue
if threshold:
th = threshold
else:
beta_a, beta_b = row.FAS[0]['betadist'][0:2]
th = scipy.stats.beta.isf(self.Pf, beta_a, beta_b, 0, 1)
if th > .9:
th, Pftemp = self._approxThld(beta_a, beta_b, sta,
row, self.Pf, 1000, 3,
backupThreshold)
msg = ('Scipy.stats.beta.isf failed with pf=%e, '
'approximated threshold to %f with a Pf of %e '
'for station %s %s using forward grid search' %
(self.Pf, th, Pftemp, sta, row.Name))
detex.log(__name__, msg, level='warning')
self.singles[sta]['Threshold'][ind] = th
def _approxThld(self, beta_a, beta_b, sta, row, target, numint, numloops,
backupThreshold):
"""
Because scipy.stats.beta.isf can break, if it returns a value near 1
when this is obviously wrong initialize grid search algorithm to get
close to desired threshold using forward problem which seems to work
where inverse fails See this bug report:
https://github.com/scipy/scipy/issues/4677
"""
startVal, stopVal = 0, 1
loops = 0
while loops < numloops:
Xs = np.linspace(startVal, stopVal, numint)
pfs = np.array([scipy.stats.beta.sf(x, beta_a, beta_b) for x in Xs])
resids = abs(pfs - target)
minind = resids.argmin()
if minind == 0 or minind == numint - 1:
msg1 = (('Grid search for threshold failing for %s on %s, '
'set it manually or use default') % (sta, row.name))
msg2 = (('Grid search for threshold failing for %s on %s, '
'using backup %.2f') % (sta, row.name, backupThreshold))
if backupThreshold is None:
detex.log(__name__, msg1, level='error', e=ValueError)
else:
detex.log(__name__, msg2, level='warn', pri=True)
return backupThreshold, target
bestPf = pfs[minind]
bestX = Xs[minind]
startVal, stopVal = Xs[minind - 1], Xs[minind + 1]
loops += 1
return bestX, bestPf
########################### Visualization Methods
[docs] def plotThresholds(self, conDatNum, xlim=[-.01, .5], **kwargs):
"""
Function to sample the continuous data and plot the thresholds
calculated with the SVD call with a histogram of detex's best
estimate of the null space (see getFAS for more details)
Parameters
------
conDatNum : int
The number of continuous data chunks to use in the sampling,
duration of chunks defined in data fetcher
xlim : list (number, number)
The x limits on the plot (often it is useful to zoom in around 0)
**kwargs are passed to the getFAS call
"""
self.getFAS(conDatNum, **kwargs)
count = 0
for station in self.ssStations:
for ind, row in self.subspaces[station].iterrows():
beta_a, beta_b = row.FAS['betadist'][0:2]
plt.figure(count)
plt.subplot(2, 1, 1)
bins = np.mean(
[row.FAS['bins'][1:], row.FAS['bins'][:-1]], axis=0)
plt.plot(bins, row.FAS['hist'])
plt.title('Station %s %s' % (station, row.Name))
plt.axvline(row.Threshold, color='g')
beta = scipy.stats.beta.pdf(bins, beta_a, beta_b)
plt.plot(bins, beta * (max(row.FAS['hist']) / max(beta)), 'k')
plt.title('%s station %s' % (row.Name, row.Station))
plt.xlim(xlim)
plt.ylabel('Count')
plt.subplot(2, 1, 2)
bins = np.mean(
[row.FAS['bins'][1:], row.FAS['bins'][:-1]], axis=0)
plt.plot(bins, row.FAS['hist'])
plt.axvline(row.Threshold, color='g')
plt.plot(bins, beta * (max(row.FAS['hist']) / max(beta)), 'k')
plt.xlabel('Detection Statistic')
plt.ylabel('Count')
plt.semilogy()
plt.ylim(ymin=10 ** -1)
plt.xlim(xlim)
count += 1
[docs] def plotFracEnergy(self):
"""
Method to plot the fractional energy captured of by the subspace for
various dimensions of rep. Each event is plotted as a grey dotted
line, the average as a red solid line, and the chosen degree of rep.
is plotted as a solid green vertical line.
Similar to Harris 2006 Fig 8
"""
for a, station in enumerate(self.ssStations):
f = plt.figure(a + 1)
f.set_figheight(1.85 * len(self.subspaces[station]))
for ind, row in self.subspaces[station].iterrows():
if not isinstance(row.FracEnergy, dict):
msg = 'fractional energy not defiend, call SVD'
detex.log(__name__, msg, level='error')
plt.subplot(len(self.subspaces[station]), 1, ind + 1)
for event in row.Events:
plt.plot(row.FracEnergy[event], '--', color='0.6')
plt.plot(row.FracEnergy['Average'], 'r')
plt.axvline(row.NumBasis, 0, 1, color='g')
plt.ylim([0, 1.1])
plt.title('Station %s, %s' % (row.Station, row.Name))
f.subplots_adjust(hspace=.4)
f.text(0.5, 0.06, 'Dimension of Representation', ha='center')
f.text(0.04, 0.5, 'Fraction of Energy Captured',
va='center', rotation='vertical')
plt.show()
[docs] def plotAlignedEvents(self): # plot aligned subspaces in SubSpaces object
"""
Plots the aligned events for each station in each cluster.
Will trim waveforms if trim times (by pickTimes or attachPickTimes)
are defined.
"""
for a, station in enumerate(self.ssStations):
for ind, row in self.subspaces[station].iterrows():
plt.figure(figsize=[10, .9 * len(row.Events)])
# f.set_figheight(1.85 * len(row.Events))
# plt.subplot(len(self.subspaces[station]), 1, ind + 1)
events = row.Events
stKeys = row.SampleTrims.keys() # sample trim keys
for evenum, eve in enumerate(events):
# plt.subplot(len(self.subspaces[station]), 1, evenum + 1)
aliTD = row.AlignedTD[eve] # aligned wf for event eve
if 'Starttime' in stKeys and 'Endtime' in stKeys:
start = row.SampleTrims['Starttime']
stop = row.SampleTrims['Endtime']
aliwf = aliTD[start: stop]
else:
aliwf = row.AlignedTD[eve]
plt.plot(aliwf / (2 * max(aliwf)) + 1.5 * evenum, c='k')
plt.xlim([0, len(aliwf)])
plt.ylim(-1, 1.5 * evenum + 1)
plt.xticks([])
plt.yticks([])
plt.title('Station %s, %s, %d events' % (station, row.Name, len(events)))
plt.show()
[docs] def plotBasisVectors(self, onlyused=False):
"""
Plots the basis vectors selected after performing the SVD
If SVD has not been called will throw error
Parameters
------------
onlyUsed : bool
If true only the selected basis vectors will be plotted. See
SVD for how detex selects basis vectors.
If false all will be plotted (used in blue, unused in red)
"""
if not self.subspaces.values()[0].iloc[0].SVDdefined:
msg = 'SVD not performed, call SVD before plotting basis vectors'
detex.log(__name__, msg, level='error')
for subnum, station in enumerate(self.ssStations):
subsp = self.subspaces[station]
for ind, row in subsp.iterrows():
num_wfs = len(row.UsedSVDKeys) if onlyused else len(row.SVD)
keyz = row.SVD.keys()
keyz.sort(reverse=True)
keyz = keyz[:num_wfs]
plt.figure(figsize=[10, .9 * num_wfs])
for keynum, key in enumerate(keyz):
wf = row.SVD[key] / (2 * max(row.SVD[key])) - 1.5 * keynum
c = 'b' if keynum < len(row.UsedSVDKeys) else '.5'
plt.plot(wf, c=c)
plt.ylim(-1.5 * keynum - 1, 1)
plt.yticks([])
plt.xticks([])
plt.title('%s station %s' % (row.Name, row.Station))
[docs] def plotOffsetTimes(self):
"""
Function to loop through each station/subspace pair and make
histograms of offset times
"""
count = 1
for station in self.ssStations:
for ind, row in self.subspaces[station].iterrows():
if len(row.SampleTrims.keys()) < 1:
msg = 'subspaces must be trimmed before plotting offsets'
detex.log(__name__, msg, level='error')
plt.figure(count)
keys = row.Events
offsets = [row.Stats[x]['offset'] for x in keys]
plt.hist(offsets)
plt.title('%s %s' % (row.Station, row.Name))
plt.figure(count + 1)
numEvs = len(row.Events)
ranmin = np.zeros(numEvs)
ranmax = np.zeros(numEvs)
orsamps = np.zeros(numEvs)
for evenum, eve in enumerate(row.Events):
tem = self.clusters.temkey[
self.clusters.temkey.NAME == eve].iloc[0]
condat = row.AlignedTD[
eve] / max(2 * abs(row.AlignedTD[eve])) + evenum + 1
Nc, Sr = row.Stats[eve]['Nc'], row.Stats[
eve]['sampling_rate']
starTime = row.Stats[eve]['starttime']
ortime = obspy.core.UTCDateTime(tem.TIME).timestamp
orsamps[evenum] = row.SampleTrims[
'Starttime'] - (starTime - ortime) * Nc * Sr
plt.plot(condat, 'k')
plt.axvline(row.SampleTrims['Starttime'], c='g')
plt.plot(orsamps[evenum], evenum + 1, 'r*')
ran = row.SampleTrims['Endtime'] - orsamps[evenum]
ranmin[evenum] = orsamps[evenum] - ran * .1
ranmax[evenum] = row.SampleTrims['Endtime'] + ran * .1
plt.xlim(int(min(ranmin)), int(max(ranmax)))
plt.axvline(min(orsamps), c='r')
plt.axvline(max(orsamps), c='r')
count += 2
############################# Pick Times functions
[docs] def pickTimes(self, duration=30, traceLimit=15, repick=False,
subspace=True, singles=True):
"""
Calls a modified version of obspyck (https://github.com/megies/obspyck)
, a GUI for picking phases, so user can manually select start times
(trim) of unclustered and clustered events.
Triming down each waveform group to only include event phases,
and not pre and post event noise, will significantly decrease the
runtime for the subspace detection (called with detex method).
Trimming is required for singletons as any singletons without trim
times will not be used as detectors).
Parameters
--------------
duration : real number
the time after the first pick (in seconds) to trim waveforms.
The fact that the streams are multiplexed is taken into account.
If None is passed then the last pick will be used as the end time
for truncating waveforms.
traceLimit : int
Limits the number of traces that will show up to be manually
picked to traceLimit events. Avoids bogging down and/or killing
the GUI with too many events.
repick : boolean
If true repick times that already have sample trim times, else
only pick those that do not.
subspace : boolean
If true pick subspaces
singles : boolean
If true pick singletons
"""
qApp = PyQt4.QtGui.QApplication(sys.argv)
if subspace:
self._pickTimes(self.subspaces, duration, traceLimit,
qApp, repick=repick)
if singles:
self._pickTimes(self.singles, duration, traceLimit, qApp,
issubspace=False, repick=repick)
def _pickTimes(self, trdfDict, duration, traceLimit, qApp,
issubspace=True, repick=False):
"""
Function to initate GUI for picking, called by pickTimes
"""
for sta in trdfDict.keys():
for ind, row in trdfDict[sta].iterrows():
if not row.SampleTrims or repick: # if not picked or repick
# Make a modified obspy stream to pass to streamPick
st = self._makeOpStream(ind, row, traceLimit)
Pks = None # This is needed or it crashes OS X
Pks = detex.streamPick.streamPick(st, ap=qApp)
d1 = {}
for b in Pks._picks:
if b: # if any picks made
d1[b.phase_hint] = b.time.timestamp
if len(d1.keys()) > 0: # if any picks made
# get sample rate and number of chans
sr = row.Stats[row.Events[0]]['sampling_rate']
Nc = row.Stats[row.Events[0]]['Nc']
# get sample divisible by NC to keep traces aligned
fp = int(min(d1.values())) # first picked phase
d1['Starttime'] = fp - fp % Nc
# if duration paramenter is defined (it is usually
# better to leave it defined)
stime = d1['Starttime']
if duration:
etime = stime + int(duration * sr * Nc)
d1['Endtime'] = etime
d1['DurationSeconds'] = duration
else:
etime = int(max(d1.values()))
d1['Endtime'] = etime
dursecs = (etime - stime) / (sr * Nc)
d1['DurationSeconds'] = dursecs
trdfDict[sta].SampleTrims[ind] = d1
for event in row.Events: # update starttimes
sspa = trdfDict[sta]
stimeOld = sspa.Stats[ind][event]['starttime']
# get updated start time
stN = stimeOld + d1['Starttime'] / (Nc * sr)
ot = trdfDict[sta].Stats[ind][event]['origintime']
offset = stN - ot
trdfDict[sta].Stats[ind][event]['starttime'] = stN
trdfDict[sta].Stats[ind][event]['offset'] = offset
if not Pks.KeepGoing:
msg = 'aborting picking, progress saved'
detex.log(__name__, msg, pri=1)
return None
self._updateOffsets()
def _makeOpStream(self, ind, row, traceLimit):
"""
Make an obspy stream of the multiplexed data stored in main detex
DataFrame
"""
st = obspy.core.Stream()
count = 0
if 'AlignedTD' in row: # if this is a subspace
for key in row.Events:
if count < traceLimit:
tr = obspy.core.Trace(data=row.AlignedTD[key])
tr.stats.channel = key
tr.stats.network = row.Name
tr.stats.station = row.Station
st += tr
count += 1
return st
else: # if this is a single event
for key in row.Events:
tr = obspy.core.Trace(data=row.MPtd[key])
tr.stats.channel = key
tr.stats.station = row.Station
st += tr
return st
def _updateOffsets(self):
"""
Calculate offset (predicted origin times), throw out extreme
outliers using median and median scaling
"""
for sta in self.subspaces.keys():
for num, row in self.subspaces[sta].iterrows():
keys = row.Stats.keys()
offsets = [row.Stats[x]['offset'] for x in keys]
self.subspaces[sta].Offsets[
num] = self._getOffsets(np.array(offsets))
for sta in self.singles.keys():
for num, row in self.singles[sta].iterrows():
keys = row.Stats.keys()
offsets = [row.Stats[x]['offset'] for x in keys]
self.singles[sta].Offsets[
num] = self._getOffsets(np.array(offsets))
[docs] def attachPickTimes(self, pksFile='PhasePicks.csv',
function='median', defaultDuration=30):
"""
Rather than picking times manually attach a file (either csv or pkl
of pandas dataframe) with pick times. Pick time file must have the
following fields: TimeStamp, Station, Event, Phase.
This file can be created by detex.util.pickPhases. If trims are
already defined attachPickTimes will not override.
----------
pksFile : str
Path to the input file (either csv or pickle)
function : str ('Average','Max', or 'Min')
Describes how to handle selecting a common pick time for
subspace groups (each event in a subspace cannot be treated
independently as the entire group is aligned to maximize
similarity). Does not apply for singles.
mean - Trims the group to the sample corresponding to the
average of the first arriving phase
median - Trims the group to the sample corresponding to the
median of the first arriving phase
max - trim to max value of start times for group
min - trim to min value of end times for group
defaultDuration : int or None
if Int, the default duration (in seconds) to trim the signal to
starting from the first arrival in pksFile for each event or
subspace group. If None, then durations are defined by first
arriving phase (start) and last arriving phase (stop) for each
event
"""
try: # read pksFile
pks = pd.read_csv(pksFile)
except Exception:
try:
pks = pd.read_pickle(pksFile)
except Exception:
msg = ('%s does not exist, or it is not a pkl or csv file'
% pksFile)
detex.log(__name__, msg, level='error')
# get appropriate function according to ssmod
if function == 'mean':
fun = np.mean
elif function == 'max':
fun = np.max
elif function == 'min':
fun = np.min
elif function == 'median':
fun = np.median
else:
msg = ('function %s not supported, options are: mean, median, min,'
' max' % function)
detex.log(__name__, msg, level='error')
# loop through each station in cluster, get singles and subspaces
for cl in self.clusters:
sta = cl.station # current station
# Attach singles
if sta in self.singles.keys():
for ind, row in self.singles[sta].iterrows():
if len(row.SampleTrims.keys()) > 0:
continue # skip if sampletrims already defined
# get phases that apply to current event and station
con1 = pks.Event.isin(row.Events)
con2 = pks.Station == sta
pk = pks[(con1) & (con2)]
eves, starttimes, Nc, Sr = self._getStats(row)
if len(pk) > 0:
trims = self._getSampTrim(eves, starttimes, Nc, Sr, pk,
defaultDuration, fun, sta,
ind, self.singles[sta], row)
if isinstance(trims, dict):
self.singles[sta].SampleTrims[ind] = trims
self._updateOffsets()
# Attach Subspaces
if sta in self.subspaces.keys():
for ind, row in self.subspaces[sta].iterrows():
if len(row.SampleTrims.keys()) > 0:
continue # skip if sampletrims already defined
# phases that apply to current event and station
con1 = pks.Event.isin(row.Events)
con2 = pks.Station == sta
pk = pks[(con1) & (con2)]
eves, starttimes, Nc, Sr = self._getStats(row)
if len(pk) > 0:
trims = self._getSampTrim(eves, starttimes, Nc, Sr, pk,
defaultDuration, fun, sta,
ind, self.subspaces[sta], row)
if isinstance(trims, dict):
self.subspaces[sta].SampleTrims[ind] = trims
self._updateOffsets()
def _getSampTrim(self, eves, starttimes, Nc, Sr, pk, defaultDuration,
fun, sta, num, DF, row):
"""
Determine sample trims for each single or subspace
"""
# stdict={}#intialize sample trim dict
startsamps = []
stopsamps = []
secduration = []
for ev in eves: # loop through each event
p = pk[pk.Event == ev]
if len(p) < 1: # if event is not recorded skip
continue
start = p.TimeStamp.min()
startsampsEve = (start - starttimes[ev]) * (Nc * Sr)
# see if any of the samples would be trimmed too much
try: # assume is single
len_test = len(row.MPtd[ev]) < startsampsEve
except AttributeError: # this is really a subspace
len_test = len(row.AlignedTD[ev]) < startsampsEve
if len_test:
utc_start = obspy.UTCDateTime(start)
msg = (('Start samples for %s on %s exceeds avaliable data,'
'check waveform quality and ensure phase pick is for '
'the correct event. The origin time is %s and the '
'pick time is %s, Skipping attaching pick. '
) % (ev, sta, ev, str(utc_start)))
detex.log(__name__, msg, level='warn')
return
# make sure starting time is not less than 0 else set to zero
if startsampsEve < 0:
startsampsEve = 0
start = starttimes[ev]
msg = 'Start time in phase file < 0 for event %s' % ev
detex.log(__name__, msg, level='warning', pri=False)
if defaultDuration:
stop = start + defaultDuration
secduration.append(defaultDuration)
else:
stop = p.TimeStamp.max()
secduration.append(stop - start)
assert stop > start # Make sure stop is greater than start
assert stop > starttimes[ev]
endsampsEve = (stop - starttimes[ev]) * (Nc * Sr)
startsamps.append(startsampsEve)
stopsamps.append(endsampsEve)
# update stats attached to each event to reflect new start time
otime = DF.Stats[num][ev]['origintime'] # origin time
DF.Stats[num][ev]['Starttime'] = start
DF.Stats[num][ev]['offset'] = start - otime
if len(startsamps) > 0:
sSamps = int(fun(startsamps))
rSSamps = sSamps - sSamps % Nc
eSamps = int(fun(stopsamps))
rESamps = eSamps - eSamps % Nc
dursec = int(fun(secduration))
outdict = {'Starttime': rSSamps, 'Endtime': rESamps,
'DurationSeconds': dursec}
return outdict
else:
return
def _getStats(self, row):
"""
Get the sampling rate, starttime, and number of channels for
each event group
"""
eves = row.Events
sr = [np.round(row.Stats[x]['sampling_rate']) for x in eves]
if len(set(sr)) != 1:
msg = ('Events %s on Station %s have different sampling rates or '
'no sampling rates' % (row.Station, row.events))
detex.log(__name__, msg, level='error')
Nc = [row.Stats[x]['Nc'] for x in eves]
if len(set(Nc)) != 1:
msg = (('Events %s on Station %s do not have the same channels or'
' have no channels') % (row.Station, row.events))
detex.log(__name__, msg, level='error')
starttimes = {x: row.Stats[x]['starttime'] for x in eves}
return eves, starttimes, list(set(Nc))[0], list(set(sr))[0]
def _getOffsets(self, offsets, m=25.):
"""
Get offsets, reject outliers bassed on median values (accounts
for possible mismatch in events and origin times)
"""
if len(offsets) == 1:
return offsets[0], offsets[0], offsets[0]
d = np.abs(offsets - np.median(offsets))
mdev = np.median(d)
s = d / mdev if mdev else 0.
if isinstance(s, float):
offs = offsets
else:
offs = offsets[s < m]
return [np.min(offs), np.median(offs), np.max(offs)]
[docs] def getFAS(
self,
conDatNum,
LTATime=5,
STATime=0.5,
staltalimit=8.0,
useSubSpaces=True,
useSingles=False,
numBins=401,
recalc=False,
**kwargs):
"""
Function to initialize a FAS (false alarm statistic) instance, used
primarily for sampling and characterizing the null space of the
subspaces and singletons. Random samples of the continuous data are
loaded, examined for high amplitude signals with a basic STA/LTA
method, and any traces with STA/LTA ratios higher than the
staltalimit parameter are rejected. The continuous DataFetcher
already attached to the SubSpace instance will be used to get
the continuous data.
Parameters
-------------
ConDatNum : int
The number of continuous data files (by default in hour chunks)
to use.
LTATime : float
The long term average time window in seconds used for
checking continuous data
STATime : float
The short term average time window in seconds for checking
continuous data
staltalimit : int or float
The value at which continuous data gets rejected as too noisey
(IE transient signals are present)
useSubSpaces : bool
If True calculate FAS for subspaces
useSingles : bool
If True calculate FAS for singles
numBins : int
Number of bins for binning distributions (so distribution can be
loaded and plotted later)
Note
---------
The results are stored in a DataFrame for each subspace/singleton
under the "FAS" column of the main DataFrame
"""
if useSubSpaces:
self._updateOffsets() # make sure offset times are up to date
for sta in self.subspaces.keys():
# check if FAS already calculated, only recalc if recalc
fas1 = self.subspaces[sta]['FAS'][0]
if isinstance(fas1, dict) and not recalc:
msg = ('FAS for station %s already calculated, to '
'recalculate pass True to the parameter recalc' %
sta)
detex.log(__name__, msg, pri=True)
else:
self.subspaces[sta]['FAS'] = detex.fas._initFAS(
self.subspaces[sta],
conDatNum,
self.clusters,
self.cfetcher,
LTATime=LTATime,
STATime=STATime,
staltalimit=staltalimit,
numBins=numBins,
dtype=self.dtype)
if useSingles:
for sta in self.singles.keys():
for a in range(len(self.singles[sta])):
fas1 = self.singles[sta]['FAS'][a]
if isinstance(fas1, dict) and not recalc:
msg = (('FAS for singleton %d already calculated on '
'station %s, to recalculate pass True to the '
'parameter recalc') % (a, sta))
detex.log(__name__, msg, pri=True)
# skip any events that have not been trimmed
elif len(self.singles[sta]['SampleTrims'][a].keys()) < 1:
continue
else:
self.singles[sta]['FAS'][a] = detex.fas._initFAS(
self.singles[sta][a:a + 1],
conDatNum,
self.clusters,
self.cfetcher,
LTATime=LTATime,
STATime=STATime,
staltalimit=staltalimit,
numBins=numBins,
dtype=self.dtype,
issubspace=False)
[docs] def detex(self,
utcStart=None,
utcEnd=None,
subspaceDB='SubSpace.db',
trigCon=0,
triggerLTATime=5,
triggerSTATime=0,
multiprocess=False,
delOldCorrs=True,
calcHist=True,
useSubSpaces=True,
useSingles=False,
estimateMags=True,
classifyEvents=None,
eventCorFile='EventCors',
utcSaves=None,
fillZeros=False):
"""
function to run subspace detection over continuous data and store
results in SQL database subspaceDB
Parameters
------------
utcStart : str or num
An obspy.core.UTCDateTime readable object defining the start time
of the correlations if not all avaliable data are to be used
utcEnd : str num
An obspy.core.UTCDateTime readable object defining the end time
of the correlations
subspaceDB : str
Path to the SQLite database to store detections in. If it already
exists delOldCorrs parameters governs if it will be deleted before
running new detections, or appended to.
trigCon is the condition for which detections should trigger.
Once the condition is set the variable minCoef is used:
0 is based on the detection statistic threshold
1 is based on the STA/LTA of the detection statistic threshold
(Only 0 is currently supported)
triggerLTATime : number
The long term average for the STA/LTA calculations in seconds.
triggerSTATime : number
The short term average for the STA/LTA calculations in seconds.
If ==0 then one sample is used.
multiprocess : bool
Determine if each station should be forked into its own process
for potential speed ups. Currently not implemented.
delOldCorrs : bool
Determines if subspaceDB should be deleted before performing
detections. If False old database is appended to.
calcHist : boolean
If True calculates the histagram for every point of the detection
statistic vectors (all hours, stations and subspaces) by keeping a
a cumulative bin count. Only slows the detections down slightly
and can be useful for threshold sanity checks. The histograms are
then returned to the main DataFrame in the SubSpace instance
as the column histSubSpaces, and saved in the subspaceDB under the
ss_hist and sg_hists tables for subspacs and singletons.
useSubspace : bool
If True the subspaces will be used as detectors to scan
continuous data
useSingles : bool
If True the singles (events that did not cluster) will be used as
detectors to scan continuous data
estimateMags : bool
If True, magnitudes will be estimated for each detection by using
two methods. The first is using standard deviation ratios, and the
second uses projected energy ratios (see chambers et al. 2015 for
details).
classifyEvents : None, str, or DataFrame
If None subspace detectors will be run over continuous data.
Else, detex will be run over event waveforms in order to classify
events into groups bassed on which subspace they are most similar
to. In the latter case the classifyEvents argument must be a
str (path to template key like csv) or DataFrame (loaded template
key file). The same event DataFetcher attached to the cluster
object will be used to get the data. This feature is Experimental.
eventCorFile : str
A path to a new pickled DataFrame created when the eventDir option
is used. Records the highest detection statistic in the file
for each event, station, and subspace. Useful when trying to
characterize events.
utcSaves : None or list of obspy DateTime readable objects
Either none (not used) or an iterrable of objects readable by
obspy.UTCDateTime. When the detections are run if the continous
data cover a time indicated in UTCSaves then the continuous data
and detection statistic vectors,are saved to a pickled dataframe
of the name "UTCsaves.pkl". This can be useful for debugging, or
extracting the DS vector for a time of interest.
fillZeros : bool
If true fill the gaps in continuous data with 0s. If True
STA/LTA of detection statistic cannot be calculated in order to
avoid dividing by 0.
Notes
----------
The same filter and decimation parameters that were used in the
ClusterStream instance will be applied.
"""
# make sure no parameters that dont work yet are selected
if multiprocess or trigCon != 0:
msg = 'multiprocessing and trigcon other than 0 not supported'
detex.log(__name__, msg, level='error')
if os.path.exists(subspaceDB):
if delOldCorrs:
os.remove(subspaceDB)
msg = 'Deleting old subspace database %s' % subspaceDB
detex.log(__name__, msg, pri=True)
else:
msg = 'Not deleting old subspace database %s' % subspaceDB
detex.log(__name__, msg, pri=True)
if useSubSpaces: # run subspaces
TRDF = self.subspaces
# determine if subspaces are defined (ie SVD has been called)
stas = self.subspaces.keys()
sv = [all(TRDF[sta].SVDdefined) for sta in stas]
if not all(sv):
msg = 'call SVD before running subspace detectors'
detex.log(__name__, msg, level='error')
Det = _SSDetex(TRDF, utcStart, utcEnd, self.cfetcher, self.clusters,
subspaceDB, trigCon, triggerLTATime, triggerSTATime,
multiprocess, calcHist, self.dtype, estimateMags,
classifyEvents, eventCorFile, utcSaves, fillZeros)
self.histSubSpaces = Det.hist
if useSingles: # run singletons
# make sure thresholds are calcualted
self.setSinglesThresholds()
TRDF = self.singles
Det = _SSDetex(TRDF, utcStart, utcEnd, self.cfetcher, self.clusters,
subspaceDB, trigCon, triggerLTATime, triggerSTATime,
multiprocess, calcHist, self.dtype, estimateMags,
classifyEvents, eventCorFile, utcSaves, fillZeros,
issubspace=False)
self.histSingles = Det.hist
# save addational info to sql database
if useSubSpaces or useSingles:
cols = ['FREQMIN', 'FREQMAX', 'CORNERS', 'ZEROPHASE']
dffil = pd.DataFrame([self.clusters.filt], columns=cols, index=[0])
detex.util.saveSQLite(dffil, subspaceDB, 'filt_params')
# get general info on each singleton/subspace and save
ssinfo, sginfo = self._getInfoDF()
sshists, sghists = self._getHistograms(useSubSpaces, useSingles)
if useSubSpaces and ssinfo is not None:
# save subspace info
detex.util.saveSQLite(ssinfo, subspaceDB, 'ss_info')
if useSingles and sginfo is not None:
# save singles info
detex.util.saveSQLite(sginfo, subspaceDB, 'sg_info')
if useSubSpaces and sshists is not None:
# save subspace histograms
detex.util.saveSQLite(sshists, subspaceDB, 'ss_hist')
if useSingles and sghists is not None:
# save singles histograms
detex.util.saveSQLite(sghists, subspaceDB, 'sg_hist')
def _getInfoDF(self):
"""
get dataframes that have info about each subspace and single
"""
sslist = [] # list in which to put DFs for each subspace/station pair
sglist = [] # list in which to put DFs for each single/station pair
for sta in self.Stations:
if sta not in self.ssStations:
msg = 'No subspaces on station %s' % sta
detex.log(__name__, msg, pri=True)
continue
for num, ss in self.subspaces[sta].iterrows(): # write ss info
name = ss.Name
station = ss.Station
events = ','.join(ss.Events)
numbasis = ss.NumBasis
thresh = ss.Threshold
if isinstance(ss.FAS, dict) and len(ss.FAS.keys()) > 1:
b1, b2 = ss.FAS['betadist'][0], ss.FAS['betadist'][1]
else:
b1, b2 = np.nan, np.nan
cols = ['Name', 'Sta', 'Events', 'Threshold', 'NumBasisUsed',
'beta1', 'beta2']
dat = [[name, station, events, thresh, numbasis, b1, b2]]
sslist.append(pd.DataFrame(dat, columns=cols))
for sta in self.Stations:
if sta not in self.singStations:
msg = 'No singletons on station %s' % sta
detex.log(__name__, msg, pri=True)
continue
for num, ss in self.singles[sta].iterrows(): # write singles info
name = ss.Name
station = ss.Station
events = ','.join(ss.Events)
thresh = ss.Threshold
if isinstance(ss.FAS, list) and len(ss.FAS[0].keys()) > 1:
b1, b2 = ss.FAS[0]['betadist'][0], ss.FAS[0]['betadist'][1]
else:
b1, b2 = np.nan, np.nan
cols = ['Name', 'Sta', 'Events', 'Threshold', 'beta1', 'beta2']
dat = [[name, station, events, thresh, b1, b2]]
sglist.append(pd.DataFrame(dat, columns=cols))
if len(sslist) > 0:
ssinfo = pd.concat(sslist, ignore_index=True)
else:
ssinfo = None
if len(sglist) > 0:
sginfo = pd.concat(sglist, ignore_index=True)
else:
sginfo = None
return ssinfo, sginfo
def _getHistograms(self, useSubSpaces, useSingles):
"""
Pull out the histogram info for saving to database
"""
cols = ['Name', 'Sta', 'Value']
if useSubSpaces:
bins = json.dumps(self.histSubSpaces['Bins'].tolist())
dat = [['Bins', 'Bins', bins]]
sshists = [pd.DataFrame(dat, columns=cols)]
for sta in self.Stations:
if sta in self.histSubSpaces.keys():
for skey in self.histSubSpaces[sta]:
try:
vl = json.dumps(self.histSubSpaces[sta][skey].tolist())
except AttributeError:
continue
dat = [[skey, sta, vl]]
sshists.append(pd.DataFrame(dat, columns=cols))
sshist = pd.concat(sshists, ignore_index=True)
else:
sshist = None
if useSingles:
bins = json.dumps(self.histSingles['Bins'].tolist())
dat = [['Bins', 'Bins', bins]]
sghists = [pd.DataFrame(dat, columns=cols)]
for sta in self.Stations:
if sta in self.histSingles.keys():
for skey in self.histSingles[sta]:
try:
vl = json.dumps(self.histSingles[sta][skey].tolist())
except AttributeError:
pass
dat = [[skey, sta, vl]]
sghists.append(pd.DataFrame(dat, columns=cols))
sghist = pd.concat(sghists, ignore_index=True)
else:
sghist = None
return sshist, sghist
########################### Python Class Attributes
def __getitem__(self, key): # make object indexable
if isinstance(key, int):
return self.subspaces[self.ssStations[key]]
elif isinstance(key, string_types):
if len(key.split('.')) == 2:
return self.subspaces[self._stakey2[key]]
elif len(key.split('.')) == 1:
return self.subspaces[self._stakey1[key]]
else:
msg = '%s is not a station in this cluster object' % key
detex.log(__name__, msg, level='error')
else:
msg = '%s must either be a int or str of station name' % key
detex.log(__name__, msg, level='error')
def __len__(self):
return len(self.subspaces)
############ MISC
[docs] def write(self, filename='subspace.pkl'):
"""
pickle the subspace class
Parameters
-------------
filename : str
Path of the file to be created
"""
cPickle.dump(self, open(filename, 'wb'))
[docs] def printOffsets(self):
"""
Function to print out the offset min max and ranges for each
station/subpace pair
"""
for station in self.ssStations:
for num, row in self.subspaces[station].iterrows():
print('%s, %s, min=%3f, max=%3f, range=%3f' %
(row.Station, row.Name, row.Offsets[0], row.Offsets[2],
row.Offsets[2] - row.Offsets[0]))