# -*- coding: utf-8 -*- # # ** header v3.0 # This file is a part of the CaosDB Project. # # Copyright (c) 2020 IndiScale GmbH <info@indiscale.com> # Copyright (c) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # # ** end header """Test whether the crawler correctly identifies the data model problems caused by a faulty model. """ import caosdb as db from caosadvancedtools import loadFiles from caosadvancedtools.cfood import fileguide from caosadvancedtools.crawler import FileCrawler from caosadvancedtools.datamodel_problems import DataModelProblems from caosadvancedtools.guard import INSERT from caosadvancedtools.models.parser import parse_model_from_yaml from caosadvancedtools.scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood, SimulationCFood) def setup_module(): """Clear problems and remove all entities except for built-in ones.""" DataModelProblems.missing.clear() try: db.execute_query("FIND entity WITH ID > 100").delete() except Exception as delete_exc: print(delete_exc) def setup(): """No further setup required.""" setup_module() def teardown(): """Delete and clear again.""" setup_module() def test_crawler_with_data_model_problems(): """Test whether data model problems are found correctly.""" # load the files basepath = "/opt/caosdb/mnt/extroot/" pathlist = [basepath+dirname for dirname in ["ExperimentalData", "DataAnalysis", "SimulationData", "Publications"]] for path in pathlist: loadFiles.loadpath(path=path, include="", exclude="", prefix="", dryrun=False, forceAllowSymlinks=False) # load and damage the model model = parse_model_from_yaml("model.yml") model.sync_data_model(noquestion=True) deleted_entities = {"Experiment", "Poster", "results"} for ent in deleted_entities: db.execute_query("FIND "+ent).delete() # Do the crawling def access(x): return "extroot" + x fileguide.access = access crawl_path = '/' files = FileCrawler.query_files(crawl_path) c = FileCrawler(files=files, use_cache=False, abort_on_exception=False, interactive=False, hideKnown=False, cfood_types=[ExperimentCFood, AnalysisCFood, PublicationCFood, SimulationCFood, ]) c.crawl(security_level=INSERT, path=crawl_path) # There should be datamodel problems assert len(DataModelProblems.missing) > 0 # Deleted entities should have been identified: assert DataModelProblems.missing.issubset(deleted_entities)