204 lines
9.3 KiB
Python
204 lines
9.3 KiB
Python
import csv
|
|
import numpy as np
|
|
import fnmatch
|
|
import datetime
|
|
|
|
|
|
def read_wdate(path,dateformat,rowheader = 0,delim = '\t'):
|
|
#imports csv data from path, with header in rows to "rowheader", time in first column in "dateformat", and with delimiter "delim". Returns header and times as list and data as numpy array.
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c-1])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[1:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[1:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k+1] = 'Nan'
|
|
out[i] = np.asarray(temp[1:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[1:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[1:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k+1] = 'NaN'
|
|
out[i] = np.asarray(r[1:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
#Create time list
|
|
times = []
|
|
for r in rows:
|
|
times.append(datetime.datetime.strptime(r[0],dateformat))
|
|
|
|
|
|
return(header,out,times)
|
|
|
|
def read_wheader(path,rowheader = 0,delim = '\t'):
|
|
#imports csv data from path, with header in rows to "rowheader", and with delimiter "delim". Returns header as list and data as numpy array.
|
|
rows = []
|
|
header = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for i, row in enumerate(csvreader):
|
|
header.append(row)
|
|
if i == rowheader:
|
|
break
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
rows = rows[rowheader:]
|
|
|
|
temp = rows[0] #Check if las column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k] = 'Nan'
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k] = 'NaN'
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
|
|
return(header,out)
|
|
|
|
def read_raw_np(path,delim = '\t'): #imports csv data from path, without header, and with delimiter "delim". Returns data as numpy array.
|
|
rows = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
|
|
temp = rows[0] #Check if last column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = np.empty([len(rows),c])
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
for n,s in enumerate(temp): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
temp[n] = '0'
|
|
elif s == 'On':
|
|
temp[n] = '1'
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(temp[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
temp[k] = 'NaN'
|
|
out[i] = np.asarray(temp[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
for n,s in enumerate(r): #replace empty entries to 0, so it can be converted
|
|
if s == '' or s == 'Off':
|
|
r[n] = '0'
|
|
elif s == 'On':
|
|
r[n] = '1'
|
|
|
|
try: #if therer a non convertable string in the row replace them with NaN
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert row to np array and set in in "out" array
|
|
except:
|
|
for k,v in enumerate(r[0:c]): #find the entry that cannot be converted and set it to NaN
|
|
try:
|
|
float(v)
|
|
except:
|
|
r[k] = 'NaN'
|
|
out[i] = np.asarray(r[0:c],dtype = float) #convert the modified row to np array and set it in "out"
|
|
|
|
|
|
return(out)
|
|
|
|
def read_raw(path,delim = '\t',convert = False):
|
|
#imports csv data from path, without header, and with delimiter "delim". Can also import strings. Returns data as list. If convert = True commas will be changed to points.
|
|
#Carefull: not only converts commas in numbers, but in all strings.
|
|
rows = []
|
|
with open(path, 'r') as file:
|
|
csvreader = csv.reader(file,delimiter = delim)
|
|
for row in csvreader:
|
|
rows.append(row)
|
|
|
|
temp = rows[0] #Check if last column is empty and adjust number of colums
|
|
if temp[-1] == '':
|
|
c = len(temp)-1
|
|
else:
|
|
c = len(temp)
|
|
out = [len(rows),c]
|
|
|
|
if len(fnmatch.filter(rows[-1],'*,*')) > 0 and convert == True: #Convert comma to point in numbers if necassary
|
|
for i,r in enumerate(rows):
|
|
temp = [s.replace(',','.') for s in r]
|
|
out[i] = temp[0:c]
|
|
else:
|
|
for i,r in enumerate(rows):
|
|
out[i] = r[0:c]
|
|
|
|
return(out) |