Source code for detex.getdata

# -*- coding: utf-8 -*-
"""
Created on Thu Nov 10 20:21:46 2015

@author: derrick
"""
from __future__ import print_function, absolute_import, unicode_literals, division

import glob
import itertools
import json
import os
import random

import numpy as np
import obspy
import pandas as pd
from six import string_types

import detex
# client imports
import obspy.clients.fdsn
import obspy.clients.neic
import obspy.clients.earthworm

conDirDefault = 'ContinuousWaveForms'
eveDirDefault = 'EventWaveForms'

# extension key to map obspy output type to extension. Add more here
formatKey = {'mseed': 'msd', 'pickle': 'pkl', 'sac': 'sac', 'Q': 'Q'}


[docs]def read(path): """ function to read a file from a path. If IOError or TypeError simply try appending os.set to start """ try: st = obspy.read(path) except (IOError, TypeError): try: st = obspy.read(os.path.join(os.path.sep, path)) except (IOError, TypeError): msg = 'Cannot read %s, the file may be corrupt, skipping it' % path detex.log(__name__, msg, level='warn', pri=True) return None return st
[docs]def quickFetch(fetch_arg, **kwargs): """ Instantiate a DataFetcher using as little information as possible. Parameters ---------- fetch_arg : str or DataFetcher instance fetch_arg can be one of three things: 1. An instance of DataFetcher 2. A valid DataFetcher Method other than dir 3. A path to a directory containing waveform data fetch_arg is checked in that order, so if you are trying to use a data directory make sure it does not share names with a valid DataFetcher method kwargs are passed to the DataFetcher class, see DataFetcher docs for details Returns ------- An instance of DataFetcher Notes -------- For client methods (eg 'uuss', 'iris') remove response is assumed True with the default prelim. filter. If you don't want this make a custom instance of DataFetcher. """ if isinstance(fetch_arg, DataFetcher): dat_fet = fetch_arg elif isinstance(fetch_arg, string_types): if fetch_arg in DataFetcher.supMethods: if fetch_arg == 'dir': msg = 'If using method dir you must pass a path to directory' detex.log(__name__, msg, level='error') dat_fet = DataFetcher(fetch_arg, removeResponse=True, **kwargs) else: if not os.path.exists(fetch_arg): msg = 'Directory %s does not exist' % fetch_arg detex.log(__name__, msg, level='error') dat_fet = DataFetcher('dir', directoryName=fetch_arg, **kwargs) else: msg = 'Input not understood, read docs and try again' detex.log(__name__, msg, level='error') return dat_fet
[docs]def makeDataDirectories(templateKey='TemplateKey.csv', stationKey='StationKey.csv', fetch='IRIS', formatOut='mseed', templateDir=eveDirDefault, timeBeforeOrigin=1 * 60, timeAfterOrigin=4 * 60, conDir=conDirDefault, secBuf=120, conDatDuration=3600, multiPro=False, getContinuous=True, getTemplates=True, removeResponse=True, opType='VEL', prefilt=[.05, .1, 15, 20]): """ Function designed to fetch data needed for detex and store them in local directories. StationKey.csv and TemplateKey.csv indicate which events to download and for which stations. Organizes ContinuousWaveForms and EventWaveForms directories. Parameters ------------ template_key : str or pd DataFrame The path to the TemplateKey csv station_key : str or pd DataFrame The path to the station key fetch : str or FetchData instance String for method argument of FetchData class or FetchData instance formatOut : str Seismic data file format, most obspy formats acceptable, options are: 'mseed','sac','GSE2','sacxy','q','sh_asc',' slist', 'tspair','segy', 'su', 'pickle', 'h5' (h5 only if obspyh5 module installed) tempalateDir : str The name of the template directory. Using the default is recommended else the templateDir parameter will have to be set in calling most other detex functions timeBeforeOrigin: real number The time in seconds before the reported origin of each template that is downloaded. timeAfterOrigin : real number(int, float, etc.) The time in seconds to download after the origin time of each template. conDir : str The name of the continuous waveform directory. Using the default is recommended secBuf : real number (int, float, etc.) The number of seconds to download after each hour of continuous data. This might be non-zero in order to capture some detections that would normally be overlooked if data did not overlap somewhat. conDatDuration : real number (int, float, etc.) The duration of the continuous data to download in seconds. multiPro : bool If True fork several processes to get data at once, potentially much faster but a bit inconsiderate on the server hosting the data getContinuous : bool If True fetch continuous data with station and date ranges listed in the station key getTemplates : bool If True get template data with stations listed in the station key and events listed in the template key removeResponse : bool If true remove instrument response opType : str Output type after removing instrument response. Choices are: "DISP" (m), "VEL" (m/s), or "ACC" (m/s**2) prefilt : list 4 real numbers Pre-filter parameters for removing instrument response, response is flat from corners 2 to 3. """ temkey = detex.util.readKey(templateKey, 'template') stakey = detex.util.readKey(stationKey, 'station') # Check output type if formatOut not in formatKey.keys(): msg = ('%s is not an acceptable format, choices are %s' % (formatOut, formatKey.keys())) detex.log(__name__, msg, level='error') # Configure data fetcher if isinstance(fetch, detex.getdata.DataFetcher): fetcher = fetch # Make sure DataFetcher is on same page as function inputs fetcher.opType = opType fetcher.removeResponse = removeResponse fetcher.prefilt = prefilt else: fetcher = detex.getdata.DataFetcher(fetch, removeResponse=removeResponse, opType=opType, prefilt=prefilt) ## Get templates if getTemplates: msg = 'Getting template waveforms' detex.log(__name__, msg, level='info', pri=True) _getTemData(temkey, stakey, templateDir, formatOut, fetcher, timeBeforeOrigin, timeAfterOrigin) ## Get continuous data if getContinuous: msg = 'Getting continuous data' detex.log(__name__, msg, level='info', pri=True) _getConData(fetcher, stakey, conDir, secBuf, opType, formatOut, duration=conDatDuration) ## Log finish msg = "finished makeDataDirectories call" detex.log(__name__, msg, level='info', close=True, pri=True)
def _getTemData(temkey, stakey, temDir, formatOut, fetcher, tb4, taft): streamGenerator = fetcher.getTemData(temkey, stakey, tb4, taft, returnName=True, temDir=temDir, skipIfExists=True) for st, name in streamGenerator: netsta = st[0].stats.network + '.' + st[0].stats.station fname = netsta + '.' + name + '.' + formatKey[formatOut] fdir = os.path.join(temDir, name) if not os.path.exists(fdir): os.makedirs(fdir) st.write(os.path.join(fdir, fname), formatOut) if not os.path.exists(os.path.join(temDir, '.index.db')): indexDirectory(temDir) def _getConData(fetcher, stakey, conDir, secBuf, opType, formatOut, duration=3600): streamGenerator = fetcher.getConData(stakey, secBuf, returnName=True, conDir=conDir, skipIfExists=True, duration=duration) for st, path, fname in streamGenerator: if st is not None: # if data were returned if not os.path.exists(path): os.makedirs(path) fname = fname + '.' + formatKey[formatOut] st.write(os.path.join(path, fname), formatOut) if not os.path.exists(os.path.join(conDir, '.index.db')): indexDirectory(conDir)
[docs]class DataFetcher(object): """ \n Class to handle data acquisition Parameters ---------- method : str or int One of the approved methods for getting data as supported by detex Options are: "dir" : A data directory as created by makeDataDirectories "client" : an obspy client can be passed to get data useful if using an in-network database "iris" : an iris client is initiated, also uses IRIS for inventory "uuss" : A client attached to the university of utah seismograph stations is initiated using CWB for waveforms and IRIS is used for station inventories client : An obspy client object Client object used to get data, from obspy.clients removeResponse : bool If True remove response before returning stream. inventoryArg : None, obspy client object, or obspy Inventory object A seperate client for station inventories, only used if removeResponse == True, also supports keyword "iris" for iris client directoryName : str A path to the continuous waveforms directory or event waveforms directory. If None is passed default names are used (ContinuousWaveForms and EventWaveForms) opType : str Output type after removing instrument response. Choices are: "DISP" (m), "VEL" (m/s), or "ACC" (m/s**2) prefilt : list of real numbers Pre-filter parameters for removing instrument response. conDatDuration : int or float Duration for continuous data in seconds conBuff : int or float The amount of data, in seconds, to download at the end of the conDatDuration. Ideally should be equal to template length, important in order to avoid missing potential events at the end of a stream timeBeforeOrigin : int or float Seconds before origin of each event to fetch (used in getTemData) timeAfterOrigin : int or float Seconds after origin of each event to fetch (used in getTemData) checkData : bool If True apply some data checks before returning streams, can be useful for older data sets. fillZeros : bool If True fill data that are not available with 0s (provided some data are available) """ supMethods = ['dir', 'client', 'uuss', 'iris'] def __init__(self, method, client=None, removeResponse=True, inventoryArg=None, directoryName=None, opType='VEL', prefilt=[.05, .1, 15, 20], conDatDuration=3600, conBuff=120, timeBeforeOrigin=1 * 60, timeAfterOrigin=4 * 60, checkData=True, fillZeros=False): self.__dict__.update(locals()) # Instantiate all inputs self.inventory = _getInventory(inventoryArg) self._checkInputs() if self.removeResponse and self.inventory is None: if self.method == 'dir': msg = ('Cannot remove response without a valid inventoryArg, ' 'setting removeResponse to False') detex.log(__name__, msg, level='warning', pri=True) self.removeResponse = False def _checkInputs(self): if not isinstance(self.method, string_types): msg = 'method must be a string. options:\n %s' % self.supMethods detex.log(__name__, msg, level='error', e=TypeError) self.method = self.method.lower() # parameter to lowercase if not self.method in DataFetcher.supMethods: msg = ('method %s not supported. Options are:\n %s' % (self.method, self.supMethods)) detex.log(__name__, msg, level='error', e=ValueError) if self.method == 'dir': if self.directoryName is None: self.directoryName = conDirDefault dirPath = glob.glob(self.directoryName) if len(dirPath) < 1: msg = ('directory %s not found make sure path is correct' % self.directoryName) detex.log(__name__, msg, level='error', e=IOError) else: self.directory = dirPath[0] self._getStream = _loadDirectoryData elif self.method == "client": if self.client is None: msg = 'Method %s requires a valid obspy client' % self.method detex.log(__name__, msg, level='error', e=ValueError) self._getStream = _assignClientFunction(self.client) elif self.method == "iris": self.client = obspy.clients.fdsn.Client("IRIS") self._getStream = _assignClientFunction(self.client) elif self.method == 'uuss': # uuss setting self.client = obspy.clients.neic.Client('128.110.129.227') self._getStream = _assignClientFunction(self.client) self.inventory = obspy.clients.fdsn.Client('iris') # use iris for resps
[docs] def getTemData(self, temkey, stakey, tb4=None, taft=None, returnName=True, temDir=None, skipIfExists=False, skipDict=None, returnTimes=False, phases=None): """ Take detex station keys and template keys and yield stream objects of all possible combinations Parameters ---------- temkey : pd DataFrame Detex template key stakey : pd DataFrame Detex station key tb4 : None, or real number Time before origin (or first phase pick if phases is not None) taft : None or real number Time after origin (or first phase pick if phases is not None) returnName : bool If True return name of event as found in template key returnNames : bool If True return event names and template names temDir : str or None Name of template directory, used to check if exists skipIfExists : bool If True dont return if file is in temDir skipDict : dict Dictionary of stations (keys, net.sta) and events (values) to skip returnTimes : bool If True return times of data phases : None, str, or DataFrame If not None must be a path to a phasePick file, in the same format as detex.util.pickPhases, or a path to a saved csv of the same. tb4 and taft will be referenced to the first arrival for each event and station, or the origin if none are available. Yields -------- Stream objects of possible combination if data are fetchable and event names if returnName == True or times of data if returnTimes == True """ if tb4 is None: tb4 = self.timeBeforeOrigin if taft is None: taft = self.timeAfterOrigin if skipDict is not None and len(skipDict.keys()) < 1: skipDict = None stakey = detex.util.readKey(stakey, key_type='station') temkey = detex.util.readKey(temkey, key_type='template') if phases is not None: phases = detex.util.readKey(phases, "phases") indexiter = itertools.product(stakey.index, temkey.index) # iter through each station/event pair and fetch data for stain, temin in indexiter: ser = temkey.loc[temin].combine_first(stakey.loc[stain]) netsta = ser.NETWORK + '.' + ser.STATION # Skip event/station combos in skipDict if skipDict is not None and netsta in skipDict.keys(): vals = skipDict[netsta] if ser.NAME in vals: continue # skip events that already have files if skipIfExists: pfile = glob.glob(os.path.join(temDir, ser.NAME, netsta + '*')) if len(pfile) > 0: continue if isinstance(ser.TIME, string_types) and 'T' in ser.TIME: time = ser.TIME else: time = float(ser.TIME) net = ser.NETWORK sta = ser.STATION chan = ser.CHANNELS.split('-') # if phases option is used then find first phase and use it if phases is not None: con1 = (phases.Event == ser.NAME) con2 = (phases.Station == '%s.%s' % (net, sta)) curEve = phases[con1 & con2] if len(curEve) < 1: # if event station pair not in phases msg = (('%s on %s was not in phase file, using origin') % (ser.NAME, sta)) detex.log(__name__, msg, level='info') t = obspy.UTCDateTime(time) else: utcs = [obspy.UTCDateTime(x) for x in curEve.TimeStamp] t = min(utcs) else: t = obspy.UTCDateTime(time) start = t - tb4 end = t + taft st = self.getStream(start, end, net, sta, chan, '??') if st is None: # skip if returns nothing continue if returnName: yield st, ser.NAME elif returnTimes: yield st, start, end else: yield st
[docs] def getConData(self, stakey, secBuff=None, returnName=False, returnTimes=False, conDir=None, skipIfExists=False, utcstart=None, utcend=None, duration=None, randSamps=None): """ Get continuous data defined by the stations and time range in the station key Parameters ----------- stakey : str or pd.DataFrame A path to the stationkey or a loaded DF of the stationkey secBuff : int A buffer in seconds to add to end of continuous data chunk so that consecutive files overlap by secBuf returnName : bool If True return the name of the file and expected path CondDir : str Path to Continuous data directory if it exists. Used to check if a file already exists so it can be skipped if skipIfExists skipIfExists : bool If True files already exists wont be downloaded again utcstart : None, int, str or obspy.UTCDateTime instance An object readable by obspy.UTCDateTime class which is the start time of continuous data to fetch. If None use time in station key utcend : None, int or str, or obspy.UTCDateTime instance An object readable by obspy.UTCDateTime class which is the end time of continuous data to fetch. If None use time in station key duration : None, int, or float The duration of each continuous data chunk to fetch, if None use conDataDuration attribute of DataFetcher instance randSamps : None or int If not None, return random number of traces rather than whole range Yields -------- Obspy trace and other requested parameters """ stakey = detex.util.readKey(stakey, 'station') if secBuff is None: secBuff = self.conBuff if duration is None: duration = self.conDatDuration for num, ser in stakey.iterrows(): netsta = ser.NETWORK + '.' + ser.STATION if utcstart is None: ts1 = obspy.UTCDateTime(ser.STARTTIME) else: ts1 = utcstart if utcend is None: ts2 = obspy.UTCDateTime(ser.ENDTIME) else: ts2 = utcend utcs = _divideIntoChunks(ts1, ts2, duration, randSamps) for utc in utcs: if conDir is not None: path, fil = _makePathFile(conDir, netsta, utc) if skipIfExists: pfile = glob.glob(os.path.join(path, fil + '*')) if len(pfile) > 0: # if already exists then skip continue start = utc end = utc + self.conDatDuration + secBuff net = ser.NETWORK sta = ser.STATION chan = ser.CHANNELS.split('-') st = self.getStream(start, end, net, sta, chan, '*') if st is None or len(st) < 1: continue if not utcend is None: if utcend.timestamp < st[0].stats.endtime.timestamp: # trim if needed st.trim(endtime=utcend) if len(st) < 1: continue if returnName and returnTimes: path, fname = _makePathFile(conDir, netsta, utc) yield st, path, fname, start, end elif returnName: path, fname = _makePathFile(conDir, netsta, utc) yield st, path, fname elif returnTimes: yield st, start, end else: yield st
[docs] def getStream(self, start, end, net, sta, chan='???', loc='??'): """ function for getting data.\n Parameters ---------- start : obspy.UTCDateTime object Start time to fetch end : obspy.UTCDateTime object End time to fetch net : str Network code, usually 2 letters sta : str Station code chan : str or list of str (should support wildcard) Channels to fetch loc : str Location code for station Returns --------- An instance of obspy.Stream populated with requested data, or None if not available. """ # make sure start and end are UTCDateTimes start = obspy.UTCDateTime(start) end = obspy.UTCDateTime(end) # check that chan input is ok if not isinstance(chan, (list, tuple)): if not isinstance(chan, string_types): msg = 'chan must be a string or list of strings' detex.log(__name__, msg, level='error') chan = [chan] # fetch stream st = self._getStream(self, start, end, net, sta, chan, loc) # perform checks if required if self.checkData: st = _dataCheck(st, start, end) # if no data return None if st is None or len(st) < 1: return None # attach response if self.removeResponse and self.inventory is not None: if not _hasResponse(st): st = _attachResponse(self, st, start, end, net, sta, loc, chan) # remove response if self.removeResponse: st = _removeInstrumentResponse(self, st) if st is None: # return None if response removal failed return None # trims and zero fills st.trim(starttime=start, endtime=end) st.merge(1) # merge and split to overwrite overlaps st = st.split() st.detrend('linear') if self.fillZeros: st.trim(starttime=start, endtime=end, pad=True, fill_value=0.0) st.merge(1, fill_value=0.0) return st
########## Functions for loading data based on selected methods ########### def _loadDirectoryData(fet, start, end, net, sta, chan, loc): """ Function to load continuous data from the detex directory structure """ # get times with slight buffer t1 = obspy.UTCDateTime(start).timestamp t2 = obspy.UTCDateTime(end).timestamp buf = 3 * fet.conDatDuration dfind = _loadIndexDb(fet.directoryName, net + '.' + sta, t1 - buf, t2 + buf) if dfind is None: t1p = obspy.UTCDateTime(t1) t2p = obspy.UTCDateTime(t2) msg = 'data from %s to %s on %s not found in %s' % (t1p, t2p, sta, fet.directoryName) detex.log(__name__, msg, level='warning', pri=False) return None # define conditions in which condata should not be loaded # con1 and con2 - No overlap (other than 10%) tra = t2 - t1 # time range con1 = ((dfind.Starttime <= t1) & (dfind.Endtime - tra * .1 < t1) & (dfind.Starttime < t2) & (dfind.Endtime < t2)) con2 = ((dfind.Starttime > t1) & (dfind.Endtime > t1) & (dfind.Starttime + tra * .1 > t2) & (dfind.Endtime >= t2)) df = dfind[~(con1 | con2)] if len(df) < 1: t1p = obspy.UTCDateTime(t1) t2p = obspy.UTCDateTime(t2) msg = 'data from %s to %s on %s not found in %s' % (t1p, t2p, sta, fet.directoryName) detex.log(__name__, msg, level='warning', pri=False) return None st = obspy.core.Stream() if len(df.Path) < 1: # if no event fits description return None for path, fname in zip(df.Path, df.FileName): fil = os.path.join(path, fname) st1 = read(fil) if not st1 is None: st += st1 # st.trim(starttime=start, endtime=end) # check if chan variable is string else iterate if isinstance(chan, string_types): stout = st.select(channel=chan) else: stout = obspy.core.Stream() for cha in chan: stout += st.select(channel=cha) loc = '*' if loc in ['???', '??'] else loc # convert ? to * stout = stout.select(location=loc) return stout def _assignClientFunction(client): """ function to take an obspy client FDSN, NEIC, EW, etc. return the correct loadFromClient function for getting data. """ if isinstance(client, obspy.clients.fdsn.Client): return _loadFromFDSN elif isinstance(client, obspy.clients.neic.Client): return _loadFromNEIC elif isinstance(client, obspy.clients.earthworm.Client): return _loadFromEarthworm else: msg = 'Client type not supported' detex.log(__name__, msg, level='error', e=TypeError) ## load from client functions, this is needed because the APIs are not the same def _loadFromNEIC(fet, start, end, net, sta, chan, loc): """ Use obspy.neic.Client to fetch waveforms """ client = fet.client # str reps of utc objects for error messages startstr = str(start) endstr = str(end) st = obspy.Stream() for cha in chan: try: # try neic client st += client.get_waveforms(net, sta, loc, cha, start, end) except: msg = ('Could not fetch data on %s from %s to %s' % (net + '.' + sta, startstr, endstr)) detex.log(__name__, msg, level='warning', pri=False) st = None return st def _loadFromEarthworm(fet, start, end, net, sta, chan, loc): client = fet.client startstr = str(start) endstr = str(end) st = obspy.Stream() if '*' in loc or '?' in loc: # adjust for earthworm loc codes loc = '--' for cha in chan: try: # try neic client st += client.get_waveforms(net, sta, loc, cha, start, end) except: msg = ('Could not fetch data on %s from %s to %s' % (net + '.' + sta, startstr, endstr)) detex.log(__name__, msg, level='warning', pri=False) st = None return st def _loadFromFDSN(fet, start, end, net, sta, chan, loc): """ Use obspy.clients.fdsn.Client to fetch waveforms """ client = fet.client # str reps of utc objects for error messages startstr = str(start) endstr = str(end) # convert channels to correct format (list seperated by ,) if not isinstance(chan, string_types): chan = ','.join(chan) else: if '-' in chan: chan = ','.join(chan.split('-')) # try to get waveforms, else return None try: st = client.get_waveforms(net, sta, loc, chan, start, end, attach_response=fet.removeResponse) except: msg = ('Could not fetch data on %s from %s to %s' % (net + '.' + sta, startstr, endstr)) detex.log(__name__, msg, level='warning', pri=False) st = None return st ########## MISC functions ############# def _attachResponse(fet, st, start, end, net, sta, loc, chan): """ Function to attach response from inventory or client """ if not fet.removeResponse or fet.inventory is None: return st if isinstance(fet.inventory, obspy.core.inventory.Inventory): st.attach_response(fet.inventory) else: inv = obspy.core.inventory.Inventory([], 'detex') for cha in chan: inv += fet.inventory.get_stations(starttime=start, endtime=end, network=net, station=sta, loc=loc, channel=cha, level="response") st.attach_response(inv) return st def _getInventory(invArg): """ Take a string, Obspy client, or inventory object and return inventory object used to attach responses to stream objects for response removal """ if isinstance(invArg, string_types): if invArg.lower() == 'iris': invArg = obspy.clients.fdsn.Client('IRIS') elif not os.path.exists(invArg): msg = ('if inventoryArg is str then it must be a client name, ie ' 'IRIS, or a path to a station xml') detex.log(__name__, msg, level='error') else: return obspy.read_inventory(invArg) elif isinstance(invArg, obspy.station.inventory.Inventory): return invArg elif isinstance(invArg, obspy.clients.fdsn.Client): return invArg elif invArg is None: return None def _dataCheck(st, start, end): # if none or empty return None if st is None or len(st) < 1: return None netsta = st[0].stats.network + '.' + st[0].stats.station time = str(st[0].stats.starttime).split('.')[0] # check if data range is way off what was requested utcmin = min([x.stats.starttime for x in st]) utcmax = max([x.stats.endtime for x in st]) if (end - start) - (utcmax - utcmin) > 60 * 10: # give 10 mine tolerance msg = '%s starting on %s is shorter than expected' % (netsta, time) detex.log(__name__, msg, pri=True) # Check sample rates if any([tr.stats.sampling_rate % 1 != 0 for tr in st]): for tr in st: tr.stats.sampling_rate = np.round(tr.stats.sampling_rate) msg = ('Found non-int sampling_rates, rounded to nearest \ int on %s around %s' % (netsta, time)) detex.log(__name__, msg, level='warning') if any([not np.any(x.data) for x in st]): msg = ('At least one channel is all 0s on %s around %s, skipping' % (netsta, time)) detex.log(__name__, msg, level='warn', pri=True) return None return st def _hasResponse(st): """ Test if all channels have responses of a stream, return bool """ return all([hasattr(tr.stats, 'response') for tr in st]) def _removeInstrumentResponse(fet, st): if not fet.removeResponse: # pass stream back if no response removal return st st.detrend('linear') # detrend st = _fftprep(st) try: st.remove_response(output=fet.opType, pre_filt=fet.prefilt) except: utc1 = str(st[0].stats.starttime).split('.')[0] utc2 = str(st[0].stats.endtime).split('.')[0] msg = 'RemoveResponse Failed for %s,%s, from %s to %s, skipping' % ( st[0].stats.network, st[0].stats.station, utc1, utc2) detex.log(__name__, msg, level='warning', pri=True) st = None return st def _fftprep(st): data = st[0].data "data is numpy vector, makes sure it is not of odd length or fft drags" if len(data) % 2 != 0 and len(data) % 100 > 50: data = np.insert(data, 0, data[0]) st[0].data = data st[0].stats.starttime = st[0].stats.starttime - st[0].stats.delta elif len(data) % 2 != 0 and len(data) % 100 < 50: data = data[1:] st[0].data = data st[0].stats.starttime = st[0].stats.starttime + st[0].stats.delta return st def _divideIntoChunks(utc1, utc2, duration, randSamps): """ Function to take two utc date time objects and create a generator to yield all time in between by intercals of duration. If randSamps is not None it will return a random subsample, still divisible by randSamps to make loading files easier. The randSamps parameter can at most rep. Inputs can be any obspy readable format """ utc1 = obspy.UTCDateTime(utc1) utc2 = obspy.UTCDateTime(utc2) # convert to time stamps (epoch time) ts1 = utc1.timestamp - utc1.timestamp % duration ts2 = utc2.timestamp - utc2.timestamp % duration if randSamps is None: t = ts1 while t <= ts2: yield obspy.UTCDateTime(t) # yield a value t += duration # add an hour else: utcList = np.arange(utc1.timestamp, utc2.timestamp, duration) if randSamps > len(utcList) / 4: msg = ('Population too small for %d random samples, taking %d' % ( randSamps, len(utcList))) detex.log(__name__, msg, level='info') randSamps = len(utcList) ranutc = random.sample(utcList, randSamps) rsamps = [obspy.UTCDateTime(x) for x in ranutc] for samp in rsamps: yield samp def _makePathFile(conDir, netsta, utc): """ Make the expected filename to see if continuous data chunk exists """ utc = obspy.UTCDateTime(utc) year = '%04d' % utc.year jd = '%03d' % utc.julday hr = '%02d' % utc.hour mi = '%02d' % utc.minute se = '%02d' % utc.second path = os.path.join(conDir, netsta, year, jd) fname = netsta + '.' + year + '-' + jd + 'T' + '-'.join([hr, mi, se]) return path, fname ###### Index directory functions ##########
[docs]def indexDirectory(dirPath): """ Create an index (.index.db) for a directory with stored waveform files which also contains quality info of each file Parameters __________ dirPath : str The path to the directory containing waveform data (any structure) """ columns = ['Path', 'FileName', 'Starttime', 'Endtime', 'Gaps', 'Nc', 'Nt', 'Duration', 'Station'] df = pd.DataFrame(columns=columns) # DataFrame for indexing msg = 'indexing, or updating index for %s' % dirPath detex.log(__name__, msg, level='info', pri=True) # Create a list of possible path permutations to save space in database pathList = [] # A list of lists with different path permutations for dirpath, dirname, filenames in os.walk(dirPath): dirList = os.path.abspath(dirpath).split(os.path.sep) # Expand pathList if needed while len(dirList) > len(pathList): pathList.append([]) # loop and put info in pathList that isnt already there for ind, value in enumerate(dirList): if not isinstance(value, list): value = [[value]] for val in value: for va in val: if va not in pathList[ind]: pathList[ind].append(va) # Loop over file names perform quality checks for fname in filenames: if fname[0] == '.': continue fpath = os.path.join(*dirList) fullpath = os.path.join(fpath, fname) qualDict = _checkQuality(fullpath) if qualDict is None: # If file is not obspy readable msg = 'obspy failed to read %s , skipping' % fullpath detex.log(__name__, msg, level='warning', pri=True) continue # skip to next file pathInts = [pathList[num].index(x) for num, x in enumerate(dirList)] df.loc[len(df), 'Path'] = json.dumps(pathInts) for key, value in qualDict.iteritems(): df.loc[len(df) - 1, key] = value df.loc[len(df) - 1, 'FileName'] = fname # Create path index key if len(pathList) < 1: msg = 'No obspy readable files found in %s' % dirPath detex.log(__name__, msg, level='error') dfInd = _createIndexDF(pathList) detex.util.saveSQLite(df, os.path.join(dirPath, '.index.db'), 'ind') detex.util.saveSQLite(dfInd, os.path.join(dirPath, '.index.db'), 'indkey')
def _createIndexDF(pathList): indLength = len(pathList) colLength = max([len(x) for x in pathList]) ind = [x for x in range(indLength)] cols = ['col_' + str(x) for x in range(colLength)] df = pd.DataFrame(index=ind, columns=cols) df.fillna(value='', inplace=True) for ind1, pl in enumerate(pathList): for ind2, item in enumerate(pl): df.loc[ind1, 'col_' + str(ind2)] = item return df def _checkQuality(stPath): """ load a path to an obspy trace and check quality """ st = read(stPath) if st is None: return None lengthStream = len(st) gaps = st.get_gaps() gapsum = np.sum([x[-2] for x in gaps]) starttime = min([x.stats.starttime.timestamp for x in st]) endtime = max([x.stats.endtime.timestamp for x in st]) duration = endtime - starttime nc = len(list(set([x.stats.channel for x in st]))) netsta = st[0].stats.network + '.' + st[0].stats.station outDict = {'Gaps': gapsum, 'Starttime': starttime, 'Endtime': endtime, 'Duration': duration, 'Nc': nc, 'Nt': lengthStream, 'Station': netsta} return outDict def _loadIndexDb(dirPath, station, t1, t2): indexFile = glob.glob(os.path.join(dirPath, '.index.db')) if len(indexFile) < 1: msg = '%s is not currently indexed, indexing now' % dirPath detex.log(__name__, msg, level='info', pri=True) indexDirectory(dirPath) indexFile = glob.glob(os.path.join(dirPath, '.index.db')) sql = (('SELECT %s FROM %s WHERE Starttime>=%f AND ' + 'Endtime<=%f AND Station="%s"') % ('*', 'ind', t1, t2, station)) df = detex.util.loadSQLite(indexFile[0], 'ind', sql=sql, silent=False) if df is None or len(df) < 1: # if not in database return None dfin = detex.util.loadSQLite(indexFile[0], 'indkey', convertNumeric=False) dfin.columns = [int(x.split('_')[1]) for x in dfin.columns] dfin.index = [int(x) for x in dfin.index] # reconstruct path df['Path'] = [_associatePathList(x, dfin) for x in df['Path']] df.sort_values(by='FileName', inplace=True) df.reset_index(drop=True, inplace=True) return df def _associatePathList(pathList, dfin): pl = json.loads(pathList) pat = [] for num, p in enumerate(pl): pat.append(dfin.loc[num, p]) return os.path.join(*pat) getAllData = makeDataDirectories