combine_raw_data.ipynb

Kernel: Python 3 (ipykernel)

import os
import numpy as np
import matplotlib.pyplot as plt

from scipy import signal
from scipy.stats import norm

!find 20220513 -maxdepth 1 -type f -name "*.vna" | sort | xargs readlink -f >  filenames.txt
!find 20220531 -maxdepth 1 -type f -name "*.vna" | sort | xargs readlink -f >> filenames.txt

with open('filenames.txt','r') as f:
    filenames = f.read().splitlines()

basenames  = [ os.path.basename(name) for name in filenames ]
locations  = [ basename.split('_')[2] for basename in basenames ]
elevations = [ basename.split('_')[3] for basename in basenames ]

num_samples = min([np.loadtxt(name,usecols=(0,),dtype=int).shape[0] for name in filenames])

# elev1 -> 7 cm, elev2 -> 19 cm, elev3 -> 32 cm
elevation_values = [7, 19, 32] # in mm

# loc  0 -> (x,y) = ( 100,   0) cm
# loc  1 -> (x,y) = (  50,   0) cm
# loc  2 -> (x,y) = (   0,   0) cm
# loc  3 -> (x,y) = (  -5,   0) cm
# loc  4 -> (x,y) = ( -50,   0) cm
# loc  5 -> (x,y) = (-100,   0) cm
# loc  6 -> (x,y) = (  -5,11.4) cm
# loc  7 -> (x,y) = ( -50,11.4) cm
# loc  8 -> (x,y) = (-100,11.4) cm
# loc  9 -> (x,y) = (-150,   0) cm
# loc 10 -> (x,y) = (-200,   0) cm
xpos_values = [ 1000,  500,    0,  -50, -500,-1000,  -50, -500,-1000,-1500,-2000] # in mm
ypos_values = [    0,    0,    0,    0,    0,    0,  114,  114,  114,    0,    0] # in mm

data = {}
for name, basename, loc, elv in zip(filenames, basenames, locations, elevations):
    try:
        data[loc][elv] = {}
    except KeyError:
        data[loc] = {}
        data[loc][elv] = {}
    
    loc_int = int(loc[len('loc'):])
    elv_int = int(elv[len('elev'):]) - 1 # -1 for zero indexing
    
    data[loc][elv]['x'] = xpos_values[loc_int]
    data[loc][elv]['y'] = ypos_values[loc_int]
    data[loc][elv]['z'] = elevation_values[elv_int]
    
    # load velocity data from colums 4 to 7
    # u, v, w1, w2 where we do not use w2
    # only 6 sig figs so float32 is fine
    vel = np.loadtxt(name,usecols=(4,5,6,7),dtype='float32')
    # convert from m/s to mm/s
    vel *= 1000
    
    # load snr data from columns 12 to 15
    # only use 3 sig figs so float32 is fine
    snr = np.loadtxt(name,usecols=(12,13,14,15),dtype='float32')
    
    # load cor data from columns 16 to 19
    cor = np.loadtxt(name,usecols=(16,17,18,19),dtype='int')
    
    # remove data where snr is less than 20
    #mask = snr < 20
    #vel[mask] = np.nan
    
    # remove data where correclation is less than 70
    mask = cor < 70
    vel[mask] = np.nan
    
    # shortest data file is num_samples samples, truncate for
    # simple array sizes
    data[loc][elv]['u'] = vel[:num_samples,0].copy()
    data[loc][elv]['v'] = vel[:num_samples,1].copy()
    data[loc][elv]['w'] = vel[:num_samples,2].copy()
    
    # sample at 200 Hz
    data[loc][elv]['t'] = np.arange(num_samples)/200

locations = set(locations)
elevations = set(elevations)

print(locations,elevations)

# must have all three elevations done for each location
for loc in locations:
    x_vals = set([data[loc][elv]['x'] for elv in elevations])
    x = min(x_vals) if len(x_vals) == 1 else None
    
    y_vals = set([data[loc][elv]['y'] for elv in elevations])
    y = min(y_vals) if len(y_vals) == 1 else None
    
    data[loc]['x'] = int(x)
    data[loc]['y'] = int(y)

{'loc4', 'loc5', 'loc1', 'loc2', 'loc0', 'loc10', 'loc3', 'loc7', 'loc6', 'loc9', 'loc8'} {'elev3', 'elev2', 'elev1'}

num_elevations = 3
num_directions = 3
total_cols = 1 + (len(locations) * num_elevations * num_directions)

arr = np.empty((num_samples,total_cols),dtype='float32')
# time array as zeroth column
arr[:,0] = np.arange(num_samples)/200 # same for all

locations = sorted(locations)
elevations = sorted(elevations)

for num_loc, loc in enumerate(locations):
    for num_elv, elv in enumerate(elevations):
        # like i + j*di + k*di*dj
        base = num_loc*num_elevations*num_directions + num_elv*num_directions
        pos_u = base + 1
        pos_v = base + 2
        pos_w = base + 3
        # will rotate by 180 degress such that
        # u = -u (backwards)
        # v = -v (backwards)
        # w = w (correct in original)
        arr[:,pos_u] =  -1*data[loc][elv]['u'].copy()
        arr[:,pos_v] =  -1*data[loc][elv]['v'].copy()
        arr[:,pos_w] =     data[loc][elv]['w'].copy()
        
directions = ['u','v','w']
headers = ['time(s)']
for loc in locations:
    for elv in elevations:
        for d in directions:
            x = data[loc][elv]['x']
            y = data[loc][elv]['y']
            z = data[loc][elv]['z']
            headers.append('_'.join([f"x={x}mm",f"y={y}mm",f"z={z}mm",d]))
header = ','.join(headers)
print(header)

np.savetxt("adv_data.csv",arr,delimiter=",",header=header,encoding='utf8')

time(s),x=1000mm_y=0mm_z=7mm_u,x=1000mm_y=0mm_z=7mm_v,x=1000mm_y=0mm_z=7mm_w,x=1000mm_y=0mm_z=19mm_u,x=1000mm_y=0mm_z=19mm_v,x=1000mm_y=0mm_z=19mm_w,x=1000mm_y=0mm_z=32mm_u,x=1000mm_y=0mm_z=32mm_v,x=1000mm_y=0mm_z=32mm_w,x=500mm_y=0mm_z=7mm_u,x=500mm_y=0mm_z=7mm_v,x=500mm_y=0mm_z=7mm_w,x=500mm_y=0mm_z=19mm_u,x=500mm_y=0mm_z=19mm_v,x=500mm_y=0mm_z=19mm_w,x=500mm_y=0mm_z=32mm_u,x=500mm_y=0mm_z=32mm_v,x=500mm_y=0mm_z=32mm_w,x=-2000mm_y=0mm_z=7mm_u,x=-2000mm_y=0mm_z=7mm_v,x=-2000mm_y=0mm_z=7mm_w,x=-2000mm_y=0mm_z=19mm_u,x=-2000mm_y=0mm_z=19mm_v,x=-2000mm_y=0mm_z=19mm_w,x=-2000mm_y=0mm_z=32mm_u,x=-2000mm_y=0mm_z=32mm_v,x=-2000mm_y=0mm_z=32mm_w,x=0mm_y=0mm_z=7mm_u,x=0mm_y=0mm_z=7mm_v,x=0mm_y=0mm_z=7mm_w,x=0mm_y=0mm_z=19mm_u,x=0mm_y=0mm_z=19mm_v,x=0mm_y=0mm_z=19mm_w,x=0mm_y=0mm_z=32mm_u,x=0mm_y=0mm_z=32mm_v,x=0mm_y=0mm_z=32mm_w,x=-50mm_y=0mm_z=7mm_u,x=-50mm_y=0mm_z=7mm_v,x=-50mm_y=0mm_z=7mm_w,x=-50mm_y=0mm_z=19mm_u,x=-50mm_y=0mm_z=19mm_v,x=-50mm_y=0mm_z=19mm_w,x=-50mm_y=0mm_z=32mm_u,x=-50mm_y=0mm_z=32mm_v,x=-50mm_y=0mm_z=32mm_w,x=-500mm_y=0mm_z=7mm_u,x=-500mm_y=0mm_z=7mm_v,x=-500mm_y=0mm_z=7mm_w,x=-500mm_y=0mm_z=19mm_u,x=-500mm_y=0mm_z=19mm_v,x=-500mm_y=0mm_z=19mm_w,x=-500mm_y=0mm_z=32mm_u,x=-500mm_y=0mm_z=32mm_v,x=-500mm_y=0mm_z=32mm_w,x=-1000mm_y=0mm_z=7mm_u,x=-1000mm_y=0mm_z=7mm_v,x=-1000mm_y=0mm_z=7mm_w,x=-1000mm_y=0mm_z=19mm_u,x=-1000mm_y=0mm_z=19mm_v,x=-1000mm_y=0mm_z=19mm_w,x=-1000mm_y=0mm_z=32mm_u,x=-1000mm_y=0mm_z=32mm_v,x=-1000mm_y=0mm_z=32mm_w,x=-50mm_y=114mm_z=7mm_u,x=-50mm_y=114mm_z=7mm_v,x=-50mm_y=114mm_z=7mm_w,x=-50mm_y=114mm_z=19mm_u,x=-50mm_y=114mm_z=19mm_v,x=-50mm_y=114mm_z=19mm_w,x=-50mm_y=114mm_z=32mm_u,x=-50mm_y=114mm_z=32mm_v,x=-50mm_y=114mm_z=32mm_w,x=-500mm_y=114mm_z=7mm_u,x=-500mm_y=114mm_z=7mm_v,x=-500mm_y=114mm_z=7mm_w,x=-500mm_y=114mm_z=19mm_u,x=-500mm_y=114mm_z=19mm_v,x=-500mm_y=114mm_z=19mm_w,x=-500mm_y=114mm_z=32mm_u,x=-500mm_y=114mm_z=32mm_v,x=-500mm_y=114mm_z=32mm_w,x=-1000mm_y=114mm_z=7mm_u,x=-1000mm_y=114mm_z=7mm_v,x=-1000mm_y=114mm_z=7mm_w,x=-1000mm_y=114mm_z=19mm_u,x=-1000mm_y=114mm_z=19mm_v,x=-1000mm_y=114mm_z=19mm_w,x=-1000mm_y=114mm_z=32mm_u,x=-1000mm_y=114mm_z=32mm_v,x=-1000mm_y=114mm_z=32mm_w,x=-1500mm_y=0mm_z=7mm_u,x=-1500mm_y=0mm_z=7mm_v,x=-1500mm_y=0mm_z=7mm_w,x=-1500mm_y=0mm_z=19mm_u,x=-1500mm_y=0mm_z=19mm_v,x=-1500mm_y=0mm_z=19mm_w,x=-1500mm_y=0mm_z=32mm_u,x=-1500mm_y=0mm_z=32mm_v,x=-1500mm_y=0mm_z=32mm_w

!cp "adv_data.csv" "$HOME/depot/Sam/Rivermouth_Tsunami/With_Flow_Experiments/Prepared_Data"