#!/usr/bin/python3 ####### # -*- coding: utf-8 -*- # ============= RCS Header ===================== #$Header: $ #$Rev$ Revision of last commit #$Author: $ Author of last commit #$Date: $ Date of last commit #$Revision: $ Revision number #$RCSfile: $ Name of the RCS file in the repository # ============ End of RCS Header # ===================== # Exemple de fabrication d'un NetCDF # Bibliotheques Python import sys from datetime import datetime import os import getopt import numpy as np # Permet d'appliquer des fonctions mathematiques usuelles a des listes # Support NetCDF4 import netCDF4 as nc4 # Reading CSV Files import csv import math # pandas DataFrame import pandas as pd # ---------------------------------------------------------------------------------------------- # --------------------- Conversion des dates / From ML ------------------- # ---------------------------------------------------------------------------------------------- # # # transforme une date de mesure en objet date et la convertir en format numerique # # il faut que les dates soient au format objet datetime # def convert_datenum(dateHeure): # print("value=",dateHeure) date,heure=dateHeure.split('T') yyyy,mm,jj=date.split('-') hh,mn,ss=heure.split(':') # print(yyyy,mm,jj,hh,mn,ss) dateobs=datetime(int(yyyy),int(mm),int(jj),int(hh),int(mn),int(ss)) return dateobs # ---------------------------------------------------------------------------------------------- # --------------------- export as NetCDF4 pour une station --------------- # ---------------------------------------------------------------------------------------------- def genereNetCDF4(fileName): fName=sys._getframe().f_code.co_name today=datetime.now() print(fName," : Generate NetCDF Data for ",fileName," Today Date =",today) dateHeure=[] tempMer=[] tempAir=[] pressionMer=[] pressionAir=[] latitude=43.3297 longitude= 5.042020 nl=0 NC_TIME_FMT = 'seconds since 2000-01-01 00:00:00 UTC' # Extract data info from CSV for stName # stationName,LocalDateTime,UTCDateTime,tz,wP-aP,wP,PT100,wT,aP,aT,condCorrected,condRaw,fileName,tel,GSMSignal,BatteryPercentage,BatteryVoltage stName=os.path.basename(fileName) # =========== Extraction des donnees du fichier CSV ================ # Lecture "dictionnaire"; La premiere ligne du fichier CSV contient les libellés des colonnes nl=0 nlS=0 longL=150 sizeF=os.path.getsize(fileName) dispL=int(sizeF/longL/10) print("File Size : ",sizeF," octets") df = pd.read_csv(fileName,skiprows=3) df['UTCDateTime']=pd.to_datetime(df['UTCDateTime']) df.sort_values(by="UTCDateTime", axis=0, ascending=True,inplace=True, na_position='first') print(df) # ======= Create NetCDF File ========== print("lat=",latitude,"lng=",longitude,"stName=",stName) stName,ext=stName.split('.') netCDFFileName="./"+stName+".nc" print ("\n\tATTENTION : GENERATE netcdf UNDER ./\n") print("Generate NetCDF File = "+netCDFFileName) f = nc4.Dataset(netCDFFileName,'w', format='NETCDF4') #'w' stands for write #==== Add global attributes ======= # Exemple 1 : Metadata Info in CSV file globalAtt=pd.read_csv("Metadata-HTMNET.csv",delimiter='=',header=None,index_col=0) indexList=globalAtt.index for index in indexList: value = globalAtt.loc[index].values[0] if index == 'Conventions': f.setncattr(index, value) else: # Seul l'attribut Conventions doit avoir une majuscule, les autres sont en minuscule # casefold retourne une chaine en minuscules # print("index value",index," , ",value) f.setncattr(index.casefold(), value) # Exemple 2 : Direct METADATA in code f.created = "Created file : " + today.strftime("%d/%m/%y") f.history = "fichier produit par programme python" f.title = "HTMNET Data for station "+stName print("Header OK") # ==== Creating Dimension ==== f.createDimension('time', None) f.createDimension('station_name_len', 20) # ==== Creating Variables ====== # time = nom variable Netcdf qui depend de la dimension "time" # Variable associee à la dimension et proprietes de la variable time = f.createVariable('time', 'i4', 'time') #Add local attributes to variable instances time.units = NC_TIME_FMT time.standard_name = "time" time.calendar = "standard" time.CoordinateAxisType = "Time" time.axis = "T" time.coverage_content_type = "coordinate" #print(df['UTCDateTime']) # voir https://unidata.github.io/cftime/api.html#cftime.date2num # il faut convertir la date en format numerique avec date2num timeVector=[] for ind in df.index: timeValue=nc4.date2num(df['UTCDateTime'][ind],units=NC_TIME_FMT,has_year_zero=False,calendar='standard') timeVector.append(timeValue) time[:]=timeVector # ==== Creating Variables and attributes ====== # Station Name station_name = f.createVariable('station_name', 'S1', 'station_name_len') station_name.long_name="station_name" station_name.cf_role = "timeseries_id" # station_name=stName dataName=np.array(stName,dtype='S20') # dataName=numpy.array(["portCros"],dtype="S1') station_name[:]=nc4.stringtochar(dataName) # latitude Station lat = f.createVariable('lat', 'f4') lat.standard_name = "latitude" lat.long_name = "station latitude" lat.units = "degrees_north" # lat.latitude = lat lat[:]=float(latitude) # longitude Station lon = f.createVariable('lon', 'f4') lon.standard_name = "longitude" lon.long_name = "station longitude" lon.units = "degrees_east" # lon.longitude = lng lon[:]=float(longitude) # Definition variable Temperature de l'eau tmer = f.createVariable('tmer', 'f4', 'time') tmer.standard_name = "sea_surface_temperature" ; tmer.long_name = "SST" tmer.units = "degC" # set variable value tmer[:]=df['wT'] # Pression à 1 metre pmer = f.createVariable('pmer', 'f4', 'time') pmer.standard_name="sea_water_pressure" pmer.long_name = "1m pressure" pmer.units = "hPa" # set variable value # pmer[:]=pressionMer pmer[:]=df['wP'] # Temperature air tair = f.createVariable('tair', 'f4', 'time') tair.standard_name = "air_temperature" ; tair.long_name = "Air Temperature" tair.units = "degC" # set variable value tair[:]=tempAir tair[:]=df['aT'] # Pression air pair = f.createVariable('pair', 'f4', 'time') pair.standard_name = "air_pressure" ; pair.long_name = "Air pressure" pair.units = "hPa" #set variable value # pair[:]=pressionAir pair[:]=df['aP'] print("Variables OK") # Close and write to disk f.close() print("File Written & Closed / ncdump -h "+netCDFFileName+" to see header") # ---------------------------------------------------------------------------------------------- # ------------------------ Fonction pour l'usage du script ----------------- # ---------------------------------------------------------------------------------------------- def usage(): fName=sys._getframe().f_code.co_name print(fName," : \n =============== USAGE ",nomScript," ======================\n") print ("Ce programme permet de generer des fichiers NetCDF a partir des donnees CSV ") print ("\t",nomScript," -h : Affiche l'aide") print ("\t",nomScript," -n : Genere un fichier NetCDF a partir du fichier CSV indique)") # ==================================== # ============== MAIN ================= # ==================================== def main(argv): if len(sys.argv) == 1: usage() sys.exit(2) try: opts, args = getopt.getopt(argv, "n:h", ["netcdf=","help"]) except getopt.GetoptError: print("No options : Exiting...") usage() sys.exit(2) for opt, arg in opts: if opt in ("-n", "--netcdf"): fileName=str(arg) try: with open(fileName): pass except IOError: print ("Error : File ",fileName," does not exist") sys.exit(2) if opt in ("-h", "--help"): usage() sys.exit(0) # Generation des fichiers netcdf print("Call genereNetCDF4(",fileName,")") genereNetCDF4(fileName) if __name__ == "__main__": # Recuperation des arguments nomScript=os.path.basename(sys.argv[0]) # unité de temps pour netcdf # print ("MonScript=",nomScript) # ---------------------- Verifie la presence d'un fichier de configuration --------------------- main(sys.argv[1:])