diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 7fa7fc1e3ba287611b84d22cd969ef50655f8a8f..876f252299991f2fa4410994b73259c3593c2198 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -25,7 +25,7 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \ RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \ cd caosdb-pylib && git checkout dev && pip3 install . # At least recommonmark 0.6 required. -RUN pip3 install recommonmark sphinx-rtd-theme +RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme COPY . /git RUN rm -r /git/.git \ && mv /git/.docker/pycaosdb.ini /git/integrationtests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea5eb78bd8323b1dd7199dc5eb91e899b1d98f81..8ebbefaa39650ddaff45b856a8a4d44a2ac495d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,20 +104,30 @@ cert: script: - cd .docker - CAOSHOSTNAME=caosdb-server ./cert.sh + style: tags: [docker] stage: style image: $CI_REGISTRY_IMAGE - needs: [] + needs: [build-testenv] script: - make style allow_failure: true +linting: + tags: [docker] + stage: style + image: $CI_REGISTRY_IMAGE + needs: [build-testenv] + script: + - make lint + allow_failure: true + unittest: tags: [docker] stage: unittest image: $CI_REGISTRY_IMAGE - needs: [] + needs: [build-testenv] script: - tox diff --git a/CHANGELOG.md b/CHANGELOG.md index c0844897a12989da9242182eaf65167b3fef31fd..19787b7d2b17a16db07e88ac7d02042a78482c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### +### Changed ### + +### Deprecated ### + +### Removed ### + +### Fixed ### + +### Security ### + +## [0.4.1] - 2022-05-03 ## +(Henrik tom Wörden) + +### Changed ### + +- `JsonSchemaParser` now identifies `name` properties in the schema with the + CaosDB name property. + +### Fixed ### + +- [#40](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/40) + `assure_object_is_in_list` now handles adding objects to an initially empty list correctly. + +## [0.4.0] - 2022-04-05 ## + +### Added ### + - CFood that creates a Record for each line in a csv file - `generic_analysis.py` allows to easily call scripts to perform analyses in server side scripting [EXPERIMENTAL] diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000000000000000000000000000000000000..44b2a5de7b1ff48da8e190a8b0f9a50ef58733cb --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,13 @@ +# Features + +## Stable +To be filled. + +## Experimental + +- `generic_analysis.py` allows to easily call scripts to perform analyses in + server side scripting +- Models parser can import from Json Schema files: + `models.parser.parse_model_from_json_schema(...)`. See the documentation of + `models.parser.JsonSchemaParser` for the limitations of the current + implementation. diff --git a/Makefile b/Makefile index 52ac04456cf59a24334003d4a0af9055dd3b11ec..d9b182cbd0b17490e9d81b900d6ba8cefadb1b64 100644 --- a/Makefile +++ b/Makefile @@ -36,5 +36,10 @@ unittest: pytest-3 unittests style: + pycodestyle --count src unittests --exclude=swagger_client autopep8 -ar --diff --exit-code --exclude swagger_client . .PHONY: style + +lint: + pylint --unsafe-load-any-extension=y -d all -e E,F --ignore=swagger_client src/caosadvancedtools +.PHONY: lint diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md index e71234b8e2bc95f954ffbebdc26acf6edd8e0b2d..7592b02d8084d3a5e6419ae66b61331026f2766c 100644 --- a/RELEASE_GUIDELINES.md +++ b/RELEASE_GUIDELINES.md @@ -9,7 +9,7 @@ guidelines of the CaosDB Project * All tests are passing. * FEATURES.md is up-to-date and a public API is being declared in that document. * CHANGELOG.md is up-to-date. -* DEPENDENCIES.md is up-to-date. +* dependencies in `setup.py` are up-to-date. ## Steps diff --git a/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/README.md b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/README.md index d844a2ddf0d87d303c69b9107a366f2e34b6d03c..2057703d18dad94127037e05b3180603e9e37380 100644 --- a/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/README.md +++ b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/README.md @@ -1,6 +1,6 @@ --- responsible: Responsible, Only -description: A description of this example analysis. +description: A description of another example analysis. sources: - file: "/ExperimentalData/2010_TestProject/2019-02-03/*.dat" diff --git a/integrationtests/extroot/Software/2020NewProject0X/2020-02-03/README.md b/integrationtests/extroot/Software/2020NewProject0X/2020-02-03/README.md index a47ea6e105c20d050ddf2fdc8cd29d4685ba30bf..bd57ffe2c43fe6406672db2dd18902b8269569d4 100644 --- a/integrationtests/extroot/Software/2020NewProject0X/2020-02-03/README.md +++ b/integrationtests/extroot/Software/2020NewProject0X/2020-02-03/README.md @@ -1,7 +1,7 @@ --- responsible: - Only Responsible MPI DS -description: A description of this example analysis. +description: A description of another example analysis. sources: - file: "/ExperimentalData/2010_TestProject/2019-02-03/*.dat" diff --git a/integrationtests/extroot/Software/2020NewProject0X/2020-02-04/README.md b/integrationtests/extroot/Software/2020NewProject0X/2020-02-04/README.md index 97b7137af372c127ee01458c9844b5ff10fd464b..b55907aaa2bb3794dbe04484c025146c3c7cd101 100644 --- a/integrationtests/extroot/Software/2020NewProject0X/2020-02-04/README.md +++ b/integrationtests/extroot/Software/2020NewProject0X/2020-02-04/README.md @@ -2,7 +2,7 @@ responsible: - Some Responsible - Responsible, No, MPI DS -description: A description of this example analysis. +description: A description of another example analysis. sources: - file: "/ExperimentalData/2010_TestProject/2019-02-03/*.dat" diff --git a/integrationtests/test.sh b/integrationtests/test.sh index a142d917215eb7469faab9c66a581539ce867e4e..5bb013db6e70a3a8393e7e3b7c7993a6da6bf9b9 100755 --- a/integrationtests/test.sh +++ b/integrationtests/test.sh @@ -35,14 +35,17 @@ echo "Testing the crawler database" python3 -m pytest test_crawler_with_cfoods.py echo "make a change" cd extroot -egrep -liRZ 'A description of another example' . | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g' +egrep -liRZ 'A description of another example' . \ + | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g' # remove a file to check that this does not lead to a crawler crash -mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back +mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx \ + DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back cd .. echo "run crawler" ./crawl.py / | tee $OUT # rename the moved file -mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx +mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back \ + extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx # check whether there was something UNAUTHORIZED grep "There where unauthorized changes" $OUT # get the id of the run which is the last field of the output string @@ -59,7 +62,8 @@ fi set -e echo "Undoing previous changes to extroot content..." cd extroot -egrep -liRZ 'A description of this example' . | xargs -0 -l sed -i -e 's/A description of this example/A description of another example/g' +egrep -liRZ 'A description of this example' . \ + | xargs -0 -l sed -i -e 's/A description of this example/A description of another example/g' cd .. echo "Done." python3 test_table.py diff --git a/integrationtests/test_assure_functions.py b/integrationtests/test_assure_functions.py index 9f4e387d52f25382d18cfb21372a06346d2b5465..b1c731dbbf25f33b54fc3a005402f292525d2d05 100644 --- a/integrationtests/test_assure_functions.py +++ b/integrationtests/test_assure_functions.py @@ -1,26 +1,25 @@ #!/usr/bin/env python # encoding: utf-8 # -# ** header v3.0 # This file is a part of the CaosDB Project. # +# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com> # Copyright (C) 2021 University Medical Center Göttingen, Institute for Medical Informatics # Copyright (C) 2021 Florian Spreckelsen <florian.spreckelsen@med.uni-goettingen.de> +# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com> # -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. # -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. -# -# ** end header +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see <https://www.gnu.org/licenses/>. """Integration tests for the `assure_...` functions from `caosadvancedtools.cfood`. They mainly test the in-place updates when no `to_be_updated` is specified. @@ -90,3 +89,29 @@ def test_assure_list_in_place(): assert len(rec2.get_property(ref_rt.name).value) == 3 assert ref_rec2.id in rec2.get_property(ref_rt.name).value assert ref_rec3.id in rec2.get_property(ref_rt.name).value + + +def test_add_to_empty_list(): + """See https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/40.""" + # @author Florian Spreckelsen + # @date 2022-04-19 + referenced_rt = db.RecordType(name="TestReferencedType").insert() + list_prop = db.Property(name="TestListProp", + datatype=db.LIST(referenced_rt)).insert() + referencing_rt = db.RecordType( + name="TestReferencingType").add_property(list_prop).insert() + + db.Record(name="TestReferencedRecord").add_parent(referenced_rt).insert() + db.Record(name="TestReferencingRecord").add_parent( + referencing_rt).add_property(list_prop, value=[]).insert() + + referenced_rec = db.execute_query("FIND TestReferencedRecord", unique=True) + referencing_rec = db.execute_query( + "FIND TestReferencingRecord", unique=True) + + assure_object_is_in_list(referenced_rec, referencing_rec, list_prop.name) + + referencing_rec = db.execute_query( + "FIND TestReferencingRecord", unique=True) + assert referencing_rec.get_property(list_prop.name).value == [ + referenced_rec.id] diff --git a/pylintrc b/pylintrc index 8a12125d4b71d3df5f7866277c41ee15401a4a93..625f83ce950841f7a239538123ef7b5812fc5c5f 100644 --- a/pylintrc +++ b/pylintrc @@ -2,8 +2,18 @@ [FORMAT] # Good variable names which should always be accepted, separated by a comma -good-names=ii,rt - +good-names=ii,rt,df [TYPECHECK] -ignored-modules=etree +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules=etree,h5py,labfolder + +[MASTER] +# TODO: The max_inferred size is necessary for https://github.com/PyCQA/pylint/issues/4577, +# otherwise pandas.read_csv's return value would be inferred as TextFileReader. +init-hook= + import sys; sys.path.extend(["src/caosadvancedtools"]); + import astroid; astroid.context.InferenceContext.max_inferred = 500; + diff --git a/release.sh b/release.sh new file mode 100755 index 0000000000000000000000000000000000000000..1af097f014de6cd9eb3d3e8ba5da34aea0fe1671 --- /dev/null +++ b/release.sh @@ -0,0 +1,4 @@ +#!/bin/bash +rm -rf dist/ build/ .eggs/ +python setup.py sdist bdist_wheel +python -m twine upload -s dist/* diff --git a/setup.py b/setup.py index 98599d9a5ead13520726546c23cbe59c57242fc0..929613de35de01da98b02c77cd76b17b04784bd8 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ from setuptools import find_packages, setup ######################################################################## MAJOR = 0 -MINOR = 3 +MINOR = 4 MICRO = 2 PRE = "" # e.g. rc0, alpha.1, 0.beta-23 ISRELEASED = False diff --git a/src/caosadvancedtools/cfood.py b/src/caosadvancedtools/cfood.py index 3c2d5408ef4d857f62ce4e908f90c4ffccef4d19..4a9f955a17fc429deb6cdd10c3645700e579b4df 100644 --- a/src/caosadvancedtools/cfood.py +++ b/src/caosadvancedtools/cfood.py @@ -1,14 +1,13 @@ #!/usr/bin/env python # encoding: utf-8 # -# ** header v3.0 # This file is a part of the CaosDB Project. # # Copyright (C) 2018 Research Group Biomedical Physics, # Max-Planck-Institute for Dynamics and Self-Organization Göttingen -# Copyright (C) 2019,2020 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2019-2022 IndiScale GmbH <info@indiscale.com> # Copyright (C) 2019,2020 Henrik tom Wörden -# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com> +# Copyright (C) 2020-2022 Florian Spreckelsen <f.spreckelsen@indiscale.com> # Copyright (C) 2021 University Medical Center Göttingen, Institute for Medical Informatics # Copyright (C) 2021 Florian Spreckelsen <florian.spreckelsen@med.uni-goettingen.de> # @@ -24,8 +23,6 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. -# -# ** end header """ Defines how something that shall be inserted into CaosDB is treated. CaosDB can automatically be filled with Records based on some structure, a file @@ -415,11 +412,13 @@ def assure_object_is_in_list(obj, containing_object, property_name, datatype=datatype) # TODO: case where multiple times the same property exists is not treated - if not isinstance(containing_object.get_property(property_name).value, list): - containing_object.get_property(property_name).value = [ - containing_object.get_property(property_name).value] - containing_object.get_property(property_name).datatype = datatype - current_list = containing_object.get_property(property_name).value + list_prop = containing_object.get_property(property_name) + if list_prop.value is None: + list_prop.value = [] + elif not isinstance(list_prop.value, list): + list_prop.value = [list_prop.value] + list_prop.datatype = datatype + current_list = list_prop.value if not isinstance(obj, list): objects = [obj] @@ -674,7 +673,8 @@ def assure_has_property(entity, name, value, to_be_updated=None, tmp_value = el.value.id if isinstance(tmp_value, list): - tmp_value = [i.id if isinstance(i, db.Entity) else i for i in tmp_value] + tmp_value = [i.id if isinstance( + i, db.Entity) else i for i in tmp_value] if tmp_value == value: contained = True @@ -810,10 +810,12 @@ class RowCFood(AbstractCFood): for key, value in self.item.iteritems(): if key in self.unique_cols: continue - rec.add_property(key, value) + assure_property_is(rec, key, + value, + to_be_updated=self.to_be_updated) -class CMeal(object): +class CMeal(): """ CMeal groups equivalent items and allow their collected insertion. @@ -841,12 +843,23 @@ class CMeal(object): matching_groups = [] def __init__(self): + self.item = None + # FIXME is this only necessary, because of inconsistent use of super().__init__()? + if "match" not in self.__dict__: + self.match = None self.__class__.existing_instances.append(self) + @staticmethod + def get_re(): + raise NotImplementedError("Subclasses must implement this function.") + @classmethod def all_groups_equal(cls, m1, m2): equal = True + if m2 is None: + return False + for group in cls.matching_groups: if (group not in m1.groupdict() or group not in m2.groupdict() or @@ -878,5 +891,5 @@ class CMeal(object): if match is None: return False - else: - return self.all_groups_equal(match, self.match) + + return self.all_groups_equal(match, self.match) diff --git a/src/caosadvancedtools/cfoods/h5.py b/src/caosadvancedtools/cfoods/h5.py index 6c68edd3668fec957126aa3234a830aab98fcd25..cbf9d0baefa435b71eeaeefe63a9b018faabe7ea 100644 --- a/src/caosadvancedtools/cfoods/h5.py +++ b/src/caosadvancedtools/cfoods/h5.py @@ -124,6 +124,7 @@ class H5CFood(AbstractFileCFood): """CFood which consumes HDF5 files.""" super().__init__(*args, **kwargs) self.h5file = None + self.identifiable_root = None self.root_name = "root" self.hdf5Container = db.Container() self.em = EntityMapping() diff --git a/src/caosadvancedtools/converter/labfolder_api.py b/src/caosadvancedtools/converter/labfolder_api.py index a29d965b1598285105a06871ee1017adfdf4e222..cf57c0155a3b3970834abb2fc1058215ef7ecba8 100644 --- a/src/caosadvancedtools/converter/labfolder_api.py +++ b/src/caosadvancedtools/converter/labfolder_api.py @@ -28,7 +28,7 @@ import time import html2text import caosdb as db -from labfolder.connection import configure_connection +from labfolder.connection import configure_connection # pylint: disable=import-error class Importer(object): diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py index 87b91a52a6034e906766a56ded787416e5c0027d..0159688c7c7d59e779d576aed54b176e802fca85 100644 --- a/src/caosadvancedtools/crawler.py +++ b/src/caosadvancedtools/crawler.py @@ -392,7 +392,6 @@ class Crawler(object): for cfood in cfoods: try: cfood.create_identifiables() - self._cached_find_or_insert_identifiables(cfood.identifiables) cfood.update_identifiables() diff --git a/src/caosadvancedtools/models/parser.py b/src/caosadvancedtools/models/parser.py index 40a61c6c9dbf3273c0287827cc68974d7be716cf..ad149222b5b90671a50943dc00bc9de8074a42f1 100644 --- a/src/caosadvancedtools/models/parser.py +++ b/src/caosadvancedtools/models/parser.py @@ -566,9 +566,11 @@ class Parser(object): db.BOOLEAN]: if is_list: - value.datatype = db.LIST(db.__getattribute__(dtype)) + value.datatype = db.LIST(db.__getattribute__( # pylint: disable=no-member + dtype)) else: - value.datatype = db.__getattribute__(dtype) + value.datatype = db.__getattribute__( # pylint: disable=no-member + dtype) continue @@ -632,8 +634,9 @@ class JsonSchemaParser(Parser): return self._create_model_from_dict(model_dict) def _create_model_from_dict(self, model_dict: [dict, List[dict]]): - """Parse a dictionary read in from the model definition in a json schema and - return the Datamodel created from it. + """Parse a dictionary and return the Datamodel created from it. + + The dictionary was typically created from the model definition in a json schema file. Parameters ---------- @@ -692,6 +695,15 @@ class JsonSchemaParser(Parser): # Each element must have a specific type raise JsonSchemaDefinitionError( f"`type` is missing in element {name}.") + if name == "name": + # This is identified with the CaosDB name property as long as the + # type is correct. + if not elt["type"] == "string": + raise JsonSchemaDefinitionError( + "The 'name' property must be string-typed, otherwise it cannot " + "be identified with CaosDB's name property." + ) + return None, force_list if "enum" in elt: ent = self._treat_enum(elt, name) elif elt["type"] in JSON_SCHEMA_ATOMIC_TYPES: @@ -726,6 +738,10 @@ class JsonSchemaParser(Parser): else: name = self._stringify(key) prop_ent, force_list = self._treat_element(prop, name) + if prop_ent is None: + # Nothing to be appended since the property has to be + # treated specially. + continue importance = db.OBLIGATORY if key in required else db.RECOMMENDED if not force_list: rt.add_property(prop_ent, importance=importance) @@ -757,7 +773,7 @@ class JsonSchemaParser(Parser): def _treat_list(self, elt: dict, name: str): # @review Timm Fitschen 2022-02-30 - if not "items" in elt: + if "items" not in elt: raise JsonSchemaDefinitionError( f"The definition of the list items is missing in {elt}.") items = elt["items"] @@ -767,7 +783,7 @@ class JsonSchemaParser(Parser): datatype = db.LIST(self._get_atomic_datatype(items)) return db.Property(name=name, datatype=datatype), False if items["type"] == "object": - if not "title" in items or self._stringify(items["title"]) == name: + if "title" not in items or self._stringify(items["title"]) == name: # Property is RecordType return self._treat_record_type(items, name), True else: diff --git a/src/caosadvancedtools/pandoc_header_tools.py b/src/caosadvancedtools/pandoc_header_tools.py index 262defd2e46ea1a6fbe80ab6c476bb8f311cc9a5..e746a26ac19c00de4ee7785399ef98478472340c 100644 --- a/src/caosadvancedtools/pandoc_header_tools.py +++ b/src/caosadvancedtools/pandoc_header_tools.py @@ -136,10 +136,10 @@ it is not at the beginning, it must be preceded by a blank line. # If a header section was found: if state == 2: headerlines = [] - for l in textlines[found_1:found_2]: - l = l.replace("\t", " ") - l = l.rstrip() - headerlines.append(l) + for line in textlines[found_1:found_2]: + line = line.replace("\t", " ") + line = line.rstrip() + headerlines.append(line) # try: try: yaml_part = yaml.load("\n".join(headerlines), Loader=yaml.BaseLoader) @@ -156,7 +156,7 @@ it is not at the beginning, it must be preceded by a blank line. else: print("Adding header in: {fn}".format(fn=filename)) add_header(filename) - return _get_header(filename) + return get_header(filename) def save_header(filename, header_data): diff --git a/src/caosadvancedtools/scifolder/simulation_cfood.py b/src/caosadvancedtools/scifolder/simulation_cfood.py index ae129e6a69ce25c6698b98124e81f8bc2921b472..c8f23f1485d7a1f64dcd940552051d2e1ec5bb07 100644 --- a/src/caosadvancedtools/scifolder/simulation_cfood.py +++ b/src/caosadvancedtools/scifolder/simulation_cfood.py @@ -88,22 +88,22 @@ class SimulationCFood(AbstractFileCFood, WithREADME): self.to_be_updated, datatype=db.LIST(db.REFERENCE)) - if SOURCES.key in self.header: + if SOURCES.key in self.header: # pylint: disable=unsupported-membership-test reference_records_corresponding_to_files( record=self.simulation, recordtypes=["Experiment", "Publication", "Simulation", "Analysis"], - globs=get_glob(self.header[SOURCES.key]), + globs=get_glob(self.header[SOURCES.key]), # pylint: disable=unsubscriptable-object property_name=dm.sources, path=self.crawled_path, to_be_updated=self.to_be_updated) self.reference_files_from_header(record=self.simulation) - if REVISIONOF.key in self.header: + if REVISIONOF.key in self.header: # pylint: disable=unsupported-membership-test reference_records_corresponding_to_files( record=self.simulation, - recordtypes=[dm.Software], + recordtypes=[dm.Software], # pylint: disable=no-member property_name=dm.revisionOf, - globs=get_glob(self.header[dm.revisionOf]), + globs=get_glob(self.header[dm.revisionOf]), # pylint: disable=unsubscriptable-object path=self.crawled_path, to_be_updated=self.to_be_updated) diff --git a/src/caosadvancedtools/scifolder/withreadme.py b/src/caosadvancedtools/scifolder/withreadme.py index 8a63e1f6d90ed4e78d01f76393cc72982cdc79d4..e1968ba49799827467c7ef93a7070b7f090010fb 100644 --- a/src/caosadvancedtools/scifolder/withreadme.py +++ b/src/caosadvancedtools/scifolder/withreadme.py @@ -121,12 +121,12 @@ class WithREADME(object): @property def header(self): if self._header is None: - if self.crawled_path.lower().endswith(".md"): + if self.crawled_path.lower().endswith(".md"): # pylint: disable=no-member self._header = get_md_header( - fileguide.access(self.crawled_path)) - elif self.crawled_path.lower().endswith(".xlsx"): + fileguide.access(self.crawled_path)) # pylint: disable=no-member + elif self.crawled_path.lower().endswith(".xlsx"): # pylint: disable=no-member self._header = get_xls_header( - fileguide.access(self.crawled_path)) + fileguide.access(self.crawled_path)) # pylint: disable=no-member else: raise RuntimeError("Readme format not recognized.") self.convert_win_paths() @@ -145,7 +145,7 @@ class WithREADME(object): globs = get_glob(self.header[field.key]) files = get_files_referenced_by_field( - globs, prefix=os.path.dirname(self.crawled_path)) + globs, prefix=os.path.dirname(self.crawled_path)) # pylint: disable=no-member description = [get_description(val) for val in self.header[field.key]] @@ -160,7 +160,7 @@ class WithREADME(object): LOGGER.warn("ATTENTION: the field {} does not reference any " "known files".format(field.key)) - self.attached_filenames.extend(flat_list) + self.attached_filenames.extend(flat_list) # pylint: disable=no-member def convert_path(self, el): """ converts the path in el to unix type @@ -185,7 +185,7 @@ class WithREADME(object): return win_path_converter(el) def convert_win_paths(self): - for field in self.win_paths: + for field in self.win_paths: # pylint: disable=no-member if field in self.header: if isinstance(self.header[field], list): @@ -245,7 +245,7 @@ class WithREADME(object): references[ref_type], record, ref_type, - to_be_updated=self.to_be_updated, + to_be_updated=self.to_be_updated, # pylint: disable=no-member ) def reference_included_records(self, record, fields, to_be_updated): @@ -255,16 +255,16 @@ class WithREADME(object): for field in fields: - if field.key not in self.header: + if field.key not in self.header: # pylint: disable=no-member continue included = [] - for item in self.header[field.key]: + for item in self.header[field.key]: # pylint: disable=no-member if INCLUDE.key in item: try: included.extend( get_entity_ids_from_include_file( - os.path.dirname(self.crawled_path), + os.path.dirname(self.crawled_path), # pylint: disable=no-member item[INCLUDE.key])) except ValueError: al = logging.getLogger("caosadvancedtools") diff --git a/src/caosadvancedtools/serverside/generic_analysis.py b/src/caosadvancedtools/serverside/generic_analysis.py index 66bec8a77e55709434b4285699e2cc2f8f804894..85d0c860df75fce205c5eaad77731fc04eee9e40 100644 --- a/src/caosadvancedtools/serverside/generic_analysis.py +++ b/src/caosadvancedtools/serverside/generic_analysis.py @@ -210,5 +210,4 @@ def main(): if __name__ == "__main__": - args = _parse_arguments() - sys.exit(main(args)) + sys.exit(main()) diff --git a/src/caosadvancedtools/serverside/helper.py b/src/caosadvancedtools/serverside/helper.py index 19efc9ed2b3e99e17eb28f5c87b0a6dbc0c47499..ba75739e0fdc0a83f235db6920471afb196f4246 100644 --- a/src/caosadvancedtools/serverside/helper.py +++ b/src/caosadvancedtools/serverside/helper.py @@ -390,11 +390,11 @@ def send_mail(from_addr, to, subject, body, cc=None, bcc=None, else: caosdb_config = db.configuration.get_config() - if not "Misc" in caosdb_config or not "sendmail" in caosdb_config["Misc"]: + if "Misc" not in caosdb_config or "sendmail" not in caosdb_config["Misc"]: err_msg = ("No sendmail executable configured. " "Please configure `Misc.sendmail` " "in your pycaosdb.ini.") - raise db.ConfigurationException(err_msg) + raise db.ConfigurationError(err_msg) sendmail = caosdb_config["Misc"]["sendmail"] # construct sendmail command diff --git a/src/caosadvancedtools/table_export.py b/src/caosadvancedtools/table_export.py index bed0edc97a794dd83b2bdd7b1c0449c710c18d3f..056207a76fa01357e2269cd4cb8e9a09905d5d90 100644 --- a/src/caosadvancedtools/table_export.py +++ b/src/caosadvancedtools/table_export.py @@ -308,7 +308,7 @@ class BaseTableExporter(object): " was specified but no record is given." ) else: - if not "selector" in d: + if "selector" not in d: d["selector"] = d[QUERY].strip().split(" ")[1] # guess find function and insert if existing else: diff --git a/src/doc/conf.py b/src/doc/conf.py index 1e07336628b696a95bc821a462f3d78f3ae11df0..c7f82a99d3b287ca72ca57430b2d4b868539d39e 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -27,9 +27,9 @@ copyright = '2021, IndiScale GmbH' author = 'Daniel Hornung' # The short X.Y version -version = '0.3.2' +version = '0.4.1' # The full version, including alpha/beta/rc tags -release = '0.3.2' +release = '0.4.1' # -- General configuration --------------------------------------------------- diff --git a/tox.ini b/tox.ini index f7b5aabf37628f57e8e192dfced969541aecfe25..dde34b987b9b08bfdfc51a06dd46a9a0e0494f28 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py36, py37, py38, py39 +envlist=py36, py37, py38, py39, py310 skip_missing_interpreters = true [testenv] deps=nose diff --git a/unittests/create_filetree.py b/unittests/create_filetree.py index 6f95618dbc834c3bc140163efdc90aa51c8d5248..f80b9681163859027bb8f8c7cd6b1387bf2d378d 100644 --- a/unittests/create_filetree.py +++ b/unittests/create_filetree.py @@ -42,8 +42,6 @@ def main(folder, dry=True): if not dry: os.mkdir(series_path) for date in [datetime.today()-timedelta(days=i)-timedelta(weeks=50*ii) for i in range(10)]: - #import IPython - # IPython.embed() exp_path = os.path.join(series_path, "Exp_"+str(date.date())) print("Exp: "+os.path.basename(exp_path)) if not dry: diff --git a/unittests/json-schema-models/datamodel_name.schema.json b/unittests/json-schema-models/datamodel_name.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..c0e86028c36172d27a4523f2c08db1b413b5c19f --- /dev/null +++ b/unittests/json-schema-models/datamodel_name.schema.json @@ -0,0 +1,12 @@ +{ + "title": "Dataset", + "type": "object", + "properties": { + "name": { "type": "string", "description": "Name of this dataset" }, + "date_time": { "type": "string", "format": "date-time" }, + "date": { "type": "string", "format": "date" }, + "integer": { "type": "integer", "description": "Some integer property" }, + "boolean": { "type": "boolean" }, + "number_prop": { "type": "number", "description": "Some float property" } + } +} diff --git a/unittests/json-schema-models/datamodel_name_wrong_type.schema.json b/unittests/json-schema-models/datamodel_name_wrong_type.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..1988ad3d8cd613def36df69f5ad30fedd0a26e48 --- /dev/null +++ b/unittests/json-schema-models/datamodel_name_wrong_type.schema.json @@ -0,0 +1,12 @@ +{ + "title": "Dataset", + "type": "object", + "properties": { + "name": { "type": "boolean", "description": "Name of this dataset" }, + "date_time": { "type": "string", "format": "date-time" }, + "date": { "type": "string", "format": "date" }, + "integer": { "type": "integer", "description": "Some integer property" }, + "boolean": { "type": "boolean" }, + "number_prop": { "type": "number", "description": "Some float property" } + } +} diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py index f5125166106c4bace21121d58a025886f9b132b9..7055bc7c51962c0cbc487f29bcdacb391218a7d3 100644 --- a/unittests/test_cfood.py +++ b/unittests/test_cfood.py @@ -48,13 +48,14 @@ class ExampleCFoodMeal(AbstractFileCFood, CMeal): CMeal.__init__(self) @classmethod - def match_item(cls, item): + def match_item(cls, path): """ standard match_match, but returns False if a suitable cfood exists """ - if cls.has_suitable_cfood(item): + print(path) + if cls.has_suitable_cfood(path): return False - return re.match(cls.get_re(), item) is not None + return re.match(cls.get_re(), path) is not None def looking_for(self, crawled_file): """ standard looking_for, but returns True if the file matches all diff --git a/unittests/test_json_schema_model_parser.py b/unittests/test_json_schema_model_parser.py index 4b44f6efa1cda19c04ee13a6a50b04cefbff9177..7f47890f413dce5511cd498fe802e03a1af3be70 100644 --- a/unittests/test_json_schema_model_parser.py +++ b/unittests/test_json_schema_model_parser.py @@ -340,3 +340,19 @@ def test_list(): assert model[name].name == name assert len(model[name].parents) == 1 assert model[name].has_parent(model["license"]) + + +def test_name_property(): + model = parse_model_from_json_schema(os.path.join( + FILEPATH, "datamodel_name.schema.json")) + + dataset_rt = model["Dataset"] + assert dataset_rt.get_property("name") is None + assert "name" not in model + + with pytest.raises(JsonSchemaDefinitionError) as err: + broken = parse_model_from_json_schema(os.path.join( + FILEPATH, "datamodel_name_wrong_type.schema.json")) + assert str(err.value).startswith( + "The 'name' property must be string-typed, otherwise it cannot be identified with CaosDB's " + "name property.") diff --git a/unittests/test_table_importer.py b/unittests/test_table_importer.py index 4c7d044ef1de877cf4072034c96aca7113f75cc0..70f0f87f8706d72c386b18f54b7a9a10908eb477 100644 --- a/unittests/test_table_importer.py +++ b/unittests/test_table_importer.py @@ -80,7 +80,6 @@ class ConverterTest(unittest.TestCase): r"\this\computer,\this\computer"), ["/this/computer", "/this/computer"]) - @pytest.mark.xfail(reason="To be fixed, see Issue #34") def test_datetime(self): test_file = os.path.join(os.path.dirname(__file__), "date.xlsx") importer = XLSImporter(converters={'d': datetime_converter, @@ -218,9 +217,10 @@ class XLSImporterTest(TableImporterTest): "float_as_float": float, "int_as_float": float, "int_as_int": int, - } - ) - df = importer.read_xls(os.path.join(os.path.dirname(__file__), "data", "datatypes.xlsx")) + } + ) + df = importer.read_xls(os.path.join( + os.path.dirname(__file__), "data", "datatypes.xlsx")) assert np.issubdtype(df.loc[0, "int_as_float"], float) @@ -253,7 +253,7 @@ class CountQueryNoneConverterTest(BaseMockUpTest): '<Query string="count record" results="0">' '</Query>' '</Response>' - ) + ) def test_check_reference_field(self): self.assertRaises(ValueError, check_reference_field, "1232", "Max") @@ -268,7 +268,7 @@ class CountQuerySingleConverterTest(BaseMockUpTest): '<Query string="count record" results="1">' '</Query>' '</Response>' - ) + ) def test_check_reference_field(self): self.assertEqual(check_reference_field("1232", "Max"), diff --git a/unittests/test_yaml_model_parser.py b/unittests/test_yaml_model_parser.py index 01730cdb1690c7c4a917475d10c9035177fb58b7..a9f072b754618e38237cbf70e74c7944551f1045 100644 --- a/unittests/test_yaml_model_parser.py +++ b/unittests/test_yaml_model_parser.py @@ -291,12 +291,12 @@ A: """ model = parse_model_from_string(modeldef) self.assertEqual(len(model), 2) - for key in model.keys(): + for key, value in model.items(): if key == "A": - self.assertTrue(isinstance(model[key], db.RecordType)) + self.assertTrue(isinstance(value, db.RecordType)) elif key == "ref": - self.assertTrue(isinstance(model[key], db.Property)) - self.assertEqual(model[key].datatype, "LIST<A>") + self.assertTrue(isinstance(value, db.Property)) + self.assertEqual(value.datatype, "LIST<A>") class ExternTest(unittest.TestCase): @@ -337,7 +337,7 @@ A: # parse_str(string) with self.assertRaises(YamlDefinitionError) as yde: parse_str(string) - assert("line {}".format(line) in yde.exception.args[0]) + assert "line {}".format(line) in yde.exception.args[0] def test_define_role(): @@ -366,11 +366,11 @@ D: role: RecordType """ entities = parse_model_from_string(model) - for l, ent in (("A", "Record"), ("b", "Property"), - ("C", "RecordType"), ("D", "RecordType")): - assert l in entities - assert isinstance(entities[l], getattr(db, ent)) - assert entities[l].role == ent + for name, ent in (("A", "Record"), ("b", "Property"), + ("C", "RecordType"), ("D", "RecordType")): + assert name in entities + assert isinstance(entities[name], getattr(db, ent)) + assert entities[name].role == ent assert entities["A"].parents[0].name == "C" assert entities["A"].name == "A"