360 lines
16 KiB
Python
360 lines
16 KiB
Python
import csv
|
|
import numpy as np
|
|
import fnmatch
|
|
import datetime
|
|
|
|
|
|
def read_wdate(path,dateformat,rowheader = 0,delim = '\t'):
|
|
'''
|
|
imports csv data from path, with header in rows to "rowheader",
|
|
time in first column in "dateformat" (if dateformat == timestamp time from 1970 in seconds is used),
|
|
and with delimiter "delim". Returns header and times as list and data as numpy array.
|
|
'''
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c-1])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[1:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[1:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k+1] = 'Nan'
|
|
out[i] = np.asarray(temp[1:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[1:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[1:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k+1] = 'NaN'
|
|
out[i] = np.asarray(r[1:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
#Create time list
|
|
times = []
|
|
if dateformat == 'timestamp':
|
|
for r in rows:
|
|
times.append(datetime.datetime.utcfromtimestamp(float(r[0])))
|
|
else:
|
|
for r in rows:
|
|
times.append(datetime.datetime.strptime(r[0],dateformat))
|
|
|
|
|
|
return(header,out,times)
|
|
|
|
def read_w2dates(path,dateformat,rowheader = 0,delim = '\t'):
|
|
'''
|
|
imports csv data from path, with header in rows to "rowheader", times in
|
|
first and second column in "dateformat", and with delimiter "delim".
|
|
Returns header and times as list and data as numpy array.
|
|
'''
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c-2])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[2:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[2:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k+2] = 'Nan'
|
|
out[i] = np.asarray(temp[2:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[2:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[2:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k+2] = 'NaN'
|
|
out[i] = np.asarray(r[2:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
#Create time list
|
|
times = []
|
|
for r in rows:
|
|
time_1 = datetime.datetime.strptime(r[0],dateformat)
|
|
time_2 = datetime.datetime.strptime(r[1],dateformat)
|
|
times.append([time_1, time_2])
|
|
|
|
|
|
return(header,out,times)
|
|
|
|
def read_w3dates(path,dateformat,rowheader = 0,delim = '\t'):
|
|
'''
|
|
imports csv data from path, with header in rows to "rowheader", times in
|
|
first to third column in "dateformat", and with delimiter "delim". Returns header
|
|
and times as list and data as numpy array.
|
|
'''
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c-3])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[3:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[3:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k+3] = 'Nan'
|
|
out[i] = np.asarray(temp[3:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[3:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[3:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k+3] = 'NaN'
|
|
out[i] = np.asarray(r[3:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
#Create time list
|
|
times = []
|
|
for r in rows:
|
|
time_1 = datetime.datetime.strptime(r[0],dateformat)
|
|
time_2 = datetime.datetime.strptime(r[1],dateformat)
|
|
time_3 = datetime.datetime.strptime(r[2],dateformat)
|
|
times.append([time_1, time_2, time_3])
|
|
|
|
|
|
return(header,out,times)
|
|
|
|
def read_wheader(path,rowheader = 0,delim = '\t'):
|
|
'''imports csv data from path, with header in rows to "rowheader", and with delimiter "delim".
|
|
Returns header as list and data as numpy array.'''
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k] = 'Nan'
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k] = 'NaN'
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
|
|
return(header,out)
|
|
|
|
def read_raw_np(path,delim = '\t'):
|
|
'''
|
|
imports csv data from path, without header, and with delimiter "delim".
|
|
Returns data as numpy array.
|
|
'''
|
|
rows = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
|
|
temp = rows[0] #Check if last column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k] = 'NaN'
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k] = 'NaN'
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
|
|
|
|
return(out)
|
|
|
|
def read_raw(path,delim = '\t',convert = False):
|
|
'''
|
|
imports csv data from path, without header, and with delimiter "delim". Can also import strings.
|
|
Returns data as list. If convert = True commas will be changed to points.
|
|
Carefull: not only converts commas in numbers, but in all strings.
|
|
'''
|
|
rows = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
|
|
temp = rows[0] #Check if last column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = []
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0 and convert == True: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
out.append(temp[0:c])
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
out.append(r[0:c])
|
|
|
|
return(out)
|