diff --git a/src/openalea/rsml/hirros.py b/src/openalea/rsml/hirros.py index 1fc3e46..5301cf1 100644 --- a/src/openalea/rsml/hirros.py +++ b/src/openalea/rsml/hirros.py @@ -13,26 +13,36 @@ import openalea.rsml -def walk(dir: str, recursive=True): - """ +def walk(dir: str, recursive=True, force=False): + """ Traverse a set of directories and return rsml files. + + Notes : rsml files are defined by 61_graph*.rsml """ dir = Path(dir) - fns = None + fns = [] for rsml_file in ['61_graph_expertized.rsml', '61_graph.rsml']: - fns = dir.glob(rsml_file) - if not fns and recursive: - fns = dir.glob('*/%s'%rsml_file) - if fns: - break + if fns and force: + continue + rsmls = dir.glob(rsml_file) + if rsmls : + fns.extend(rsmls) + if recursive: + rsmls = dir.glob('*/%s'%rsml_file) + if rsmls : + fns.extend(rsmls) + if not fns: print("ERROR: no rsml files found") final_fns = [] for fn in fns: - if (fn.parent/'80_graph_analysis.xlsx').exists(): - continue - else: - final_fns.append(fn) + if not force: + if (fn.parent/'80_graph_analysis.xlsx').exists(): + continue + elif (fn.parent/'80_graph_expertized_analysis.xlsx').exists(): + continue + final_fns.append(fn) + return final_fns def read(fn): @@ -41,7 +51,7 @@ def read(fn): def times(g): """Return Observation dates in hours.""" - obs_t = g._graph_properties['metadata']['observation-hours'].split(',') + obs_t = g._graph_properties['metadata']['observation-hours'] observations = [float(t) for t in obs_t] return observations @@ -100,8 +110,7 @@ def length_and_number(secondary): return total_length.tolist(), total_number.tolist() -def write_xls(xls_file, obs, primaries, secondaries): - +def dataframes(obs, primaries, secondaries): data = [obs] index=['time(h)'] for i, p in enumerate(primaries): @@ -120,8 +129,6 @@ def write_xls(xls_file, obs, primaries, secondaries): dfs= [] - - for sec in secondaries: data = [obs] @@ -144,7 +151,10 @@ def write_xls(xls_file, obs, primaries, secondaries): df = pd.DataFrame(data, index = index) dfs.append(df) - #print(dfp) + return dfp, dfs + +def write_xls(xls_file, dfp, dfs): + if xls_file: with pd.ExcelWriter(xls_file, engine="xlsxwriter") as writer: @@ -152,15 +162,57 @@ def write_xls(xls_file, obs, primaries, secondaries): sheet_name='Prim', float_format="%.2f", header = False) + workbook = writer.book + bold_format = workbook.add_format({'bold': True}) + + worksheet = writer.sheets['Prim'] + worksheet.set_row(0, None, bold_format) + for i, df in enumerate(dfs): df.to_excel(writer, sheet_name='RP%d'%(i+1), float_format="%.2f", header = False) + worksheet = writer.sheets['RP%d'%(i+1)] + worksheet.set_row(0, None, bold_format) +def write_xls_all(xls_file, fns, primaries, secondaries): -def run(fn): - g = read(fn) + + with pd.ExcelWriter(xls_file, engine="xlsxwriter") as writer: + startrow = 0 + for i, dfp in enumerate(primaries): + + dfp.to_excel(writer, + sheet_name='Prim', + float_format="%.2f", + header = False, + startrow=startrow+1) + workbook = writer.book + bold_format = workbook.add_format({'bold': True}) + + worksheet = writer.sheets['Prim'] + worksheet.set_row(startrow+1, None, bold_format) + worksheet.write(startrow, 0, fns[i]) + startrow += len(dfp)+3 + + startrow = 0 + for i, dfs in enumerate(secondaries): + name = fns[i] + for j, df in enumerate(dfs): + df.to_excel(writer, + sheet_name='Lateral', + float_format="%.2f", + header = False, + startrow=startrow+1) + worksheet = writer.sheets['Lateral'] + worksheet.set_row(startrow+1, None, bold_format) + worksheet.write(startrow, 0, name) + worksheet.write(startrow, 1, 'Plant %d'%(j+1)) + + startrow += len(df)+3 + +def process(g): obs = times(g) plant_ids = g.vertices(scale=1) @@ -173,42 +225,94 @@ def run(fn): s2 = secondary(g, pid, obs) secondaries.append(s2) + df_primary, df_secondaries = dataframes(obs, prims, secondaries) + return df_primary, df_secondaries + +def run(fn): + g = read(fn) + dfp, dfs = process(g) + # Write in xlsx - xlsx_file = fn.parent/'80_graph_analysis.xlsx' - write_xls(xlsx_file, - obs, primaries=prims, secondaries=secondaries) + if 'expertized' in fn: + xlsx_file = fn.parent/'80_graph_expertized_analysis.xlsx' + else: + xlsx_file = fn.parent/'80_graph_analysis.xlsx' + write_xls(xlsx_file, dfp, dfs) print('WRITE %s'%xlsx_file) +def run_all(fns): + + prims = [] + secondaries = [] + + for fn in fns: + g = read(fn) + dfp, dfs = process(g) + prims.append(dfp) + secondaries.append(dfs) + + write_xls_all('gxe_results.xlsx', fns, prims, secondaries) + print('WRITE gxe_results.xlsx') + +def merge_and_compute_primaries(dataframes): + """Mean of Mean + or mean of total Length and total number""" + pass + +def merge_and_compute_secondaries(dataframes): + """Mean of Mean + or mean of total Length and total number""" + pass + def main(): parser = argparse.ArgumentParser() parser.add_argument('-d', default='.', help='directory to process') parser.add_argument('-r', default=True, type=bool, help='traverse the directories recursively') + parser.add_argument('-f', help='text file containing the list of directories to process') args = parser.parse_args() dir = args.d recursive = args.r - - fns = walk(dir=dir, recursive=recursive) - if not fns: - return - - if len(fns) == 1: - fn = fns[0] - print('Process file %s'%(fns[0])) - run(fn) - + gxe_file = args.f + + fns = [] + if gxe_file: + print('GxE file to process', gxe_file) + dirs = [] + with open(gxe_file, 'r') as gxe: + dirs = [d.strip() for d in gxe] + dirs = [d for d in dirs if Path(d).exists()] + + for d in dirs: + rsml_files = walk(dir=d, recursive=recursive, force=True) + fns.extend(rsml_files) + + print('Process files %s'%(' '.join(fns))) + run_all(fns) else: - for fn in fns: + fns = walk(dir=dir, recursive=recursive) + + if not fns: + return + + if len(fns) == 1: + fn = fns[0] print('Process file %s'%(fns[0])) run(fn) + else: + for fn in fns: + print('Process file %s'%(fns[0])) + run(fn) + if __name__=='__main__': - g = read('set_de_5/230403VS004') + fns = walk('set_de_5/230403VS004') + g = read(fns[0]) obs = times(g) plant_ids = g.vertices(scale=1) diff --git a/src/openalea/rsml/io.py b/src/openalea/rsml/io.py index 2bc1d8c..d6adb3c 100644 --- a/src/openalea/rsml/io.py +++ b/src/openalea/rsml/io.py @@ -1,4 +1,4 @@ -""" XML SmartRoot / RootNav reader and writer +""" XML SmartRoot / RootNav / RootSystemTracker reader and writer TODO: * Manage metadata @@ -18,7 +18,7 @@ """ ############################################################################## -# XML SmartRoot / RootNav reader and writer +# XML SmartRoot / RootNav / RootSystemTracker reader and writer ############################################################################## from ast import literal_eval @@ -48,7 +48,18 @@ def parse(self, filename, debug=False): root = doc.getroot() # recursive call of the functions to add neww plants/root axis to the MTG self.dispatch(root) - + + # if some functions are defined in the MTG properties but not in metadata, add them + graph = self._g + if graph.graph_properties().get('metadata', {}).get('functions') is None: + graph.graph_properties()['metadata']['functions'] = [] + if graph.properties().get('time'): + graph.graph_properties()['metadata']['functions'].append('time') + if graph.properties().get('time_hours'): + graph.graph_properties()['metadata']['functions'].append('time_hours') + if graph.properties().get('diameter'): + graph.graph_properties()['metadata']['functions'].append('diameter') + g = fat_mtg(self._g) # Add metadata as property of the graph @@ -87,6 +98,7 @@ def metadata(self, elts, **properties): meta = self._metadata = dict() gprop = self._g.graph_properties() #print([elt.tag for elt in elts]) + pixel_size = None for elt in elts: elt_tag = elt.tag #print(elt_tag) @@ -97,12 +109,21 @@ def metadata(self, elts, **properties): elif elt_tag in ['user','file-key','software','unit']: meta[elt_tag] = elt.text elif elt_tag in ["property-definitions","time-sequence","image",'private']: - #print(elt_tag) self.dispatch(elt) + elif elt_tag == "observation-hours": + elt_text = elt.text + meta[elt_tag] = [literal_eval(v) for v in elt_text.split(',') if v] + elif elt_tag in ['size', 'pixel_size']: + pixel_size = float(elt.text) # RootSystemTracker use size for pixel_size before image element D: elif elt_tag=='mtg_graph_properties': gprop.update(read_xml_tree(elt)) else: meta[elt_tag] = read_xml_tree(elt) + + if pixel_size: + # BUG : resolution now is similar to image + meta['resolution'] = pixel_size + meta.setdefault('image',{})['resolution'] = pixel_size gprop['metadata'] = meta @@ -134,7 +155,7 @@ def function_definition(self, elts, **properties): label = prop.pop('label') if label: self._propdef[label]=prop - + def time_sequence(self, elts, **properties): """ A plant with parameters and a recursive structure. @@ -218,6 +239,7 @@ def polyline(self, elts, **properties): self._polyline = [] # will store all points in `elts` self._time = [] self._time_hours = [] + # self._diameter = [] for elt in elts: self.dispatch(elt) @@ -230,13 +252,16 @@ def polyline(self, elts, **properties): if self._time_hours : self._node.time_hours = self._time_hours self._time_hours = None - + # if self._diameter : + # self._node.diameter = self._diameter + # self._diameter = None def point(self, elts, **properties): poly = self._polyline point = [] times = self._time times_hours = self._time_hours + # diameters = self._diameter if properties: if 'x' in properties or 'coord_x' in properties: coords = ['x', 'y', 'z'] @@ -250,13 +275,14 @@ def point(self, elts, **properties): coords = ['th', 'coord_th'] time_hours = [float(properties[c]) for c in coords if c in properties] times_hours.append(time_hours[0]) + # if 'diameter' in properties: + # diameter = float(properties['diameter']) + # diameters.append(diameter) else: point = [float(elt.text) for elt in elts] poly.append(point) - #print('point', point) - - + #print('point', point) def functions(self, elts, **properties): """ A root axis with geometry, functions, properties. @@ -412,18 +438,26 @@ def mtg(self): def metadata(self): g = self._g self.xml_meta = xml.SubElement(self.xml_root,'metadata') - gmetadata = metadata.set_metadata(g) for tag in metadata.flat_metadata: self.SubElement(self.xml_meta, tag=tag, text=str(gmetadata[tag])) - + # image metadata - + self.observation_hours(gmetadata) self.image(gmetadata) self.property_definitions(gmetadata) # print('TODO: time-sequence') + def observation_hours(self,metadata): + """ dump observation-hours element of metadata """ + obs = metadata.get('observation-hours') # List of observation hours + if obs is None: return + + obs_elt = self.SubElement(self.xml_meta, 'observation-hours') + txt = ','.join(str(hour) for hour in obs) + obs_elt.text = txt + def image(self,metadata): """ dump image element of metadata """ image = metadata.get('image') @@ -485,8 +519,6 @@ def scene(self): # self.process_vertex(vid) - - def plant(self, vid): g = self._g @@ -512,16 +544,17 @@ def root(self, xml_parent, mtg_vid): self.xml_nodes[vid] = axis = self.SubElement(xml_parent, 'root') # set xml attributes - props = g[vid] + props = g[vid] axis.attrib['id'] = str(props.pop('id', vid)) axis.attrib['label'] = str(props.pop('label', g.label(vid))) if 'po:accession' in props: axis.attrib['po:accession'] = str(props.pop('po:accession')) # set xml axis element - self.properties(vid, axis) - ##self.functions(axis,**props) self.geometry(axis,**props) + self.functions(axis,**props) + self.properties(vid, axis) + # process children root axis # -------------------------- @@ -583,12 +616,12 @@ def functions(self, axis, **props): for tag in pname: if tag in props: if functions_elt is None: - functions_elt = self.SubElement(xml_elt, 'functions') + functions_elt = self.SubElement(axis, 'functions') function_elt = self.SubElement(functions_elt, 'function') function_elt.attrib['domain'] = 'polyline' - function_elt.attrib['name'] = tag + function_elt.attrib['name'] = tag - for sample in attrib[tag]: + for sample in props[tag]: sample_elt = self.SubElement(function_elt, 'sample') if isinstance(sample, (tuple, list)) and len(sample) == 2: sample_elt.attrib['position'] = str(sample[0]) @@ -596,8 +629,6 @@ def functions(self, axis, **props): else: sample_elt.attrib['value'] = str(sample) - - ########################################################################## # Wrapper functions for OpenAlea usage. @@ -621,4 +652,4 @@ def mtg2rsml(g, rsml_file): with open(rsml_file, 'wb') as f: # F. Bauget 2022-04-11: with python 3 xml.tostring(self.xml_root, encoding='UTF-8') gives bytes so I open in binary mode f.write(s) else: - rsml_file.write(s) + rsml_file.write(s) \ No newline at end of file diff --git a/src/openalea/rsml/matching.py b/src/openalea/rsml/matching.py index 9848045..829eb20 100644 --- a/src/openalea/rsml/matching.py +++ b/src/openalea/rsml/matching.py @@ -20,7 +20,7 @@ def match_plants(t1,t2, max_distance=None): The matching is done usinf `one_to_one_match` """ - from operator import div + from operator import truediv as div # compute seed position of plants in t # ------------------------------------ diff --git a/src/openalea/rsml/metadata.py b/src/openalea/rsml/metadata.py index 6341c8f..e3c2791 100644 --- a/src/openalea/rsml/metadata.py +++ b/src/openalea/rsml/metadata.py @@ -25,14 +25,13 @@ function also fill missing items, folowing the specified behavior describe in the function documentation. """ -import xml.etree.ElementTree as xml - # ordered list of metadata attribute name flat_metadata = ['version','unit','resolution','software','user', 'last-modified','file-key'] metadata_names = flat_metadata + ['image', 'property-definitions', 'function-definitions', 'time-sequence', + 'observation-hours', 'private'] # default values @@ -91,9 +90,17 @@ def set_metadata(g): from os.path import getctime creation = getctime(image['name']) image['captured'] = datetime.fromtimestamp(creation).isoformat() + except KeyError: # not defined + pass except OSError: # no such file pass - + + if 'observation-hours' in metadata: + # table of observation times + obs = metadata['observation-hours'] + if isinstance(obs, str): + import ast + metadata['observation-hours'] = list(ast.literal_eval(obs)) # convert string to list if metadata['file-key']!=default['file-key']: import uuid diff --git a/test/data/UC1_1PN001_old_format.rsml b/test/data/UC1_1PN001_old_format.rsml new file mode 100644 index 0000000..688ec9f --- /dev/null +++ b/test/data/UC1_1PN001_old_format.rsml @@ -0,0 +1,2112 @@ + + +1.4 +pixel(um) +76.0 +18-07-2023 11:22:19 +RootSystemTracker +Unknown +61_graph_expertized +0.0,13.6589,19.6578,25.6575,31.6572,37.6574,43.6565,49.6553,55.6572,61.6557,67.6562,73.6558,79.6562,85.6552,91.6561,95.4315,101.4321,107.433,113.432,119.4305,125.4316,131.4299,137.43,143.4312,158.606,164.601,170.6014,176.6003,182.5998 + + +Nothing there + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/test_hirros.py b/test/test_hirros.py index a3afbe8..2ca92bc 100644 --- a/test/test_hirros.py +++ b/test/test_hirros.py @@ -26,7 +26,11 @@ def test_observations(): plant_ids = g.vertices(scale=1) prims = hirros.primaries(g, plant_ids, obs) - +def test_fix_bug_resolution(): + g = get_mtg() + meta = g._graph_properties['metadata'] + assert meta['resolution'] == 76. + assert meta['image'] != meta['resolution'] def test1(): "Returns secondary dataframe"