crawl.py

#!/usr/bin/env python3
# encoding: utf-8
#
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
#

import argparse
import logging
import sys
from argparse import RawTextHelpFormatter

import caosdb as db

from caosadvancedtools.crawler import FileCrawler
from caosadvancedtools.guard import INSERT, RETRIEVE, UPDATE, Guard
from caosadvancedtools.utils import set_log_level
from scifolder import (AnalysisCFood, ExperimentCFood, ProjectCFood,
                       PublicationCFood, SimulationCFood)


def get_parser():
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=RawTextHelpFormatter)

    parser.add_argument("path")

    return parser


def access(path):
    return "extroot" + path


if __name__ == "__main__":
    logger = logging.getLogger("caosadvancedtools")
    conlogger = logging.getLogger("connection")
    conlogger.setLevel(level=logging.ERROR)
    logger.setLevel(level=logging.WARN)

    parser = get_parser()
    args = parser.parse_args()

    logger.info("Starting query...")
    files = FileCrawler.query_files(args.path)
    logger.info("Query done...")
    config = db.configuration.get_config()
    c = FileCrawler(files=files, use_cache=True, access=access,
                    interactive=False, hideKnown=True,
                    food=[ProjectCFood,
                          ExperimentCFood, AnalysisCFood,
                          PublicationCFood, SimulationCFood,
                          ])
    c.crawl(security_level=UPDATE)