diff --git a/.gitignore b/.gitignore
index 4402aa11bc399c03400c4427c669b93ebb2637ce..182ed05e1404483ecb553c8a4e469a86a77ba27c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ src/doc/_apidoc/
start_caosdb_docker.sh
src/doc/_apidoc
/dist/
+*.egg-info
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bfac6b0012cb067657567381752a600736e7d788..67415c1b3a7da52e3179bec8463cd69ac3c667aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,10 +120,10 @@ unittest_py3.9:
script:
- tox
-unittest_py3.8:
+unittest_py3.7:
tags: [cached-dind]
stage: test
- image: python:3.8
+ image: python:3.7
script: &python_test_script
# install dependencies
- pip install pytest pytest-cov
@@ -135,12 +135,24 @@ unittest_py3.8:
- caosdb-crawler --help
- pytest --cov=caosdb -vv ./unittests
+unittest_py3.8:
+ tags: [cached-dind]
+ stage: test
+ image: python:3.8
+ script: *python_test_script
+
unittest_py3.10:
tags: [cached-dind]
stage: test
image: python:3.10
script: *python_test_script
+unittest_py3.11:
+ tags: [cached-dind]
+ stage: test
+ image: python:3.11
+ script: *python_test_script
+
inttest:
tags: [docker]
services:
@@ -277,3 +289,27 @@ style:
script:
- autopep8 -r --diff --exit-code .
allow_failure: true
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+# Based on: https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/ci/editor?branch_name=main
+pages_prepare: &pages_prepare
+ tags: [ cached-dind ]
+ stage: deploy
+ needs: []
+ image: $CI_REGISTRY/caosdb/src/caosdb-pylib/testenv:latest
+ only:
+ refs:
+ - /^release-.*$/i
+ script:
+ - echo "Deploying documentation"
+ - make doc
+ - cp -r build/doc/html public
+ artifacts:
+ paths:
+ - public
+pages:
+ <<: *pages_prepare
+ only:
+ refs:
+ - main
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02ddf17f5c0b7af95106e1e7a2b6cac3a7953a69..f086e1317f05277452659adf3fe20547adab2ae3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,47 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.3.0] - 2022-01-30 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- Identifiable class to represent the information used to identify Records.
+- Added some StructureElements: BooleanElement, FloatElement, IntegerElement,
+ ListElement, DictElement
+- String representation for Identifiables
+- [#43](https://gitlab.com/caosdb/caosdb-crawler/-/issues/43) the crawler
+ version can now be specified in the `metadata` section of the cfood
+ definition. It is checked against the installed version upon loading of the
+ definition.
+- JSON schema validation can also be used in the DictElementConverter
+- YAMLFileConverter class; to parse YAML files
+- Variables can now be substituted within the definition of yaml macros
+- debugging option for the match step of Converters
+- Re-introduced support for Python 3.7
+
+### Changed ###
+
+- Some StructureElements changed (see "How to upgrade" in the docs):
+ - Dict, DictElement and DictDictElement were merged into DictElement.
+ - DictTextElement and TextElement were merged into TextElement. The "match"
+ keyword is now invalid for TextElements.
+- JSONFileConverter creates another level of StructureElements (see "How to upgrade" in the docs)
+- create_flat_list function now collects entities in a set and also adds the entities
+ contained in the given list directly
+
+### Deprecated ###
+
+- The DictXYElements are now depricated and are now synonyms for the
+ XYElements.
+
+### Fixed ###
+
+- [#39](https://gitlab.com/caosdb/caosdb-crawler/-/issues/39) Merge conflicts in
+ `split_into_inserts_and_updates` when cached entity references a record
+ without id
+- Queries for identifiables with boolean properties are now created correctly.
+
## [0.2.0] - 2022-11-18 ##
(Florian Spreckelsen)
@@ -21,13 +62,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
DictFloatElement. This behavior can be changed (see converter documentation).
**Note** This might lead to additional matches compared to previous versions.
- `_AbstractDictElementConverter` uses `re.DOTALL` for `match_value`
-- The "fallback" parent, the name of the element in the cfood, is only used
+- The "fallback" parent, the name of the element in the cfood, is only used
when the object is created and only if there are no parents given.
-### Deprecated ###
-
-### Removed ###
-
### Fixed ###
* [#31](https://gitlab.com/caosdb/caosdb-crawler/-/issues/31) Identified cache:
@@ -39,8 +76,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
interpreted as integers and vice versa, there are defaults for allowing other
types and this can be changed per converter
-### Security ###
-
## [0.1.0] - 2022-10-11
(Florian Spreckelsen)
@@ -68,5 +103,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* FIX: #35 Parent cannot be set from value
* [#6](https://gitlab.com/caosdb/caosdb-crawler/-/issues/6): Fixed many type
hints to be compatible to python 3.8
-* [#9](https://gitlab.com/caosdb/caosdb-crawler/-/issues/9): Sclaras of types
+* [#9](https://gitlab.com/caosdb/caosdb-crawler/-/issues/9): Scalars of types
different than string can now be given in cfood definitions
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000000000000000000000000000000000000..ad00d0edb29ecfe2edf4b1aeb621ff35f8304f90
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+ - family-names: Fitschen
+ given-names: Timm
+ orcid: https://orcid.org/0000-0002-4022-432X
+ - family-names: Schlemmer
+ given-names: Alexander
+ orcid: https://orcid.org/0000-0003-4124-9649
+ - family-names: Hornung
+ given-names: Daniel
+ orcid: https://orcid.org/0000-0002-7846-6375
+ - family-names: tom Wörden
+ given-names: Henrik
+ orcid: https://orcid.org/0000-0002-5549-578X
+ - family-names: Parlitz
+ given-names: Ulrich
+ orcid: https://orcid.org/0000-0003-3058-1435
+ - family-names: Luther
+ given-names: Stefan
+ orcid: https://orcid.org/0000-0001-7214-8125
+title: CaosDB - Crawler
+version: 0.3.0
+doi: 10.3390/data4020083
+date-released: 2023-01-30
\ No newline at end of file
diff --git a/README.md b/README.md
index b97fc8775ba334c03c5c0a42238e9f396301e6b3..6c94473c066439b1645712c0046cd890b6b38715 100644
--- a/README.md
+++ b/README.md
@@ -1,34 +1,57 @@
-# caoscrawler
+# CaosDB-Crawler
-A new crawler for CaosDB.
+## Welcome
+This is the repository of the CaosDB-Crawler, a tool for automatic data
+insertion into [CaosDB](https://gitlab.com/caosdb/caosdb-meta).
-This package has yaml-header-tools as a dependency:
-https://gitlab.com/salexan/yaml-header-tools
+This is a new implementation resolving problems of the original implementation
+in [caosdb-advancedtools](https://gitlab.com/caosdb/caosdb-advanced-user-tools)
+## Setup
+Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
+setup this code.
-This python package can be installed using `pip`, e.g.:
-```bash
-pip install --user .
-```
-# Usage
+## Further Reading
-work in progress
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-crawler/) of the CaosDB-Crawler for more information.
-# Running the tests
+## Contributing
-After installation of the package run (within the project folder):
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
-```bash
-pytest
-```
+### Code of Conduct
+
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
+
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-crawler).
+* You want to contribute code?
+ * **Forking:** Please fork the repository and create a merge request in GitLab and choose this repository as
+ target. Make sure to select "Allow commits from members who can merge the target branch" under
+ Contribution when creating the merge request. This allows our team to work with you on your
+ request.
+ * **Code style:** This project adhers to the PEP8 recommendations, you can test your code style
+ using the `autopep8` tool (`autopep8 -i -r ./`). Please write your doc strings following the
+ [NumpyDoc](https://numpydoc.readthedocs.io/en/latest/format.html) conventions.
+* You can also contact us at **info (AT) caosdb.de** and join the
+ CaosDB community on
+ [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
+
+
+There is the file `unittests/records.xml` that servers as a dummy for a server state with files.
+You can recreate this by uncommenting a section in `integrationtests/basic_example/test_basic.py`
+and rerunning the integration test.
## Integration Tests
+
see `integrationtests/README.md`
-# Contributers
+## Contributers
The original authors of this package are:
@@ -36,10 +59,10 @@ The original authors of this package are:
- Henrik tom Wörden
- Florian Spreckelsen
-# License
+## License
-Copyright (C) 2021-2022 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+Copyright (C) 2021-2022 Research Group Biomedical Physics, Max Planck Institute
+ for Dynamics and Self-Organization Göttingen.
Copyright (C) 2021-2022 IndiScale GmbH
All files in this repository are licensed under a [GNU Affero General Public
diff --git a/README_SETUP.md b/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..d478016ecde09dab8820d398b15df325f4159380
--- /dev/null
+++ b/README_SETUP.md
@@ -0,0 +1 @@
+src/doc/README_SETUP.md
\ No newline at end of file
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index d6bc2c9ae41b8032a5567f786eb060d7b67d2cc5..a08eef0d90b6e35026a3b4dbeffad66738e52997 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -24,6 +24,7 @@ guidelines of the CaosDB Project
- `version` variables in `src/doc/conf.py`
- Version in [setup.cfg](./setup.cfg): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
`ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+ - `CITATION.cff` (update version and date)
5. Merge the release branch into the main branch.
diff --git a/integrationtests/basic_example/test_basic.py b/integrationtests/basic_example/test_basic.py
index a3195d4cda683b37da2ef58d8bb3e7b7c18de390..0c847b08a729f3b112cbdf3c38bac31309cda125 100755
--- a/integrationtests/basic_example/test_basic.py
+++ b/integrationtests/basic_example/test_basic.py
@@ -33,6 +33,7 @@ import argparse
import sys
from argparse import RawTextHelpFormatter
from caoscrawler import Crawler, SecurityMode
+from caoscrawler.identifiable import Identifiable
import caosdb as db
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
import pytest
@@ -106,6 +107,15 @@ def crawler_extended(ident):
return cr
+def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
+ ins, ups = crawler.synchronize()
+
+ proj = db.execute_query("FIND Project WITH identifier='SpeedOfLight'", unique=True)
+ with pytest.raises(RuntimeError, match=".*unambigiously.*"):
+ print(crawler.identifiableAdapter.retrieve_identified_record_for_identifiable(
+ Identifiable(properties={'project': proj.id})))
+
+
def test_single_insertion(clear_database, usemodel, crawler, ident):
ins, ups = crawler.synchronize()
@@ -114,14 +124,15 @@ def test_single_insertion(clear_database, usemodel, crawler, ident):
for i in reversed(range(len(res))):
if res[i].parents[0].name == "PyTestInfo":
del res[i]
- filename = rfp("..", "..", "unittests", "records.xml")
- with open(filename, "w") as f:
- xml = res.to_xml()
- # Remove noscript and transaction benchmark:
- for tag in ("noscript", "TransactionBenchmark"):
- if xml.find(tag) is not None:
- xml.remove(xml.find(tag))
- f.write(db.common.utils.xml2str(xml))
+ # uncomment this to recreate the `records.xml` file
+ # filename = rfp("..", "..", "unittests", "records.xml")
+ # with open(filename, "w") as f:
+ # xml = res.to_xml()
+ # # Remove noscript and transaction benchmark:
+ # for tag in ("noscript", "TransactionBenchmark"):
+ # if xml.find(tag) is not None:
+ # xml.remove(xml.find(tag))
+ # f.write(db.common.utils.xml2str(xml))
assert len(ins) == 18
assert len(ups) == 0
diff --git a/integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml b/integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml
index eaf2690ae130cb61c8a74452e3e4e1d4fd06846a..7a64d708667182b80b739812e5fdf3369fc5b462 100644
--- a/integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml
+++ b/integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml
@@ -32,107 +32,111 @@ Data:
match: .dataspace.json
validate: schema/dataspace.schema.json
subtree:
- dataspace_id_element:
- type: DictIntegerElement
- match_name: "dataspace_id"
- match_value: "(?P<id>[0-9]+)"
- records:
- Dataspace:
- dataspace_id: $id
- archived_element:
- type: DictBooleanElement
- match_name: "archived"
- match_value: "(?P<archived>.*)"
- records:
- Dataspace:
- archived: $archived
- url_element:
- type: DictTextElement
- match_name: "url"
- match_value: "(?P<url>.*)"
- records:
- Dataspace:
- url: $url
- coordinator_element:
- type: DictDictElement
- match_name: "coordinator"
- records:
- Person:
- parents:
- - Person
- Dataspace:
- Person: $Person
- subtree: &person_subtree
- full_name_element:
- type: DictTextElement
- match_name: "full_name"
- match_value: "(?P<full_name>.*)"
+ jsondict:
+ type: DictElement
+ match: .*
+ subtree:
+ dataspace_id_element:
+ type: IntegerElement
+ match_name: "dataspace_id"
+ match_value: "(?P<id>[0-9]+)"
records:
- Person:
- full_name: $full_name
- full_name_nonlatin_element:
- type: DictTextElement
- match_name: "full_name_nonlatin"
- match_value: "(?P<full_name_nonlatin>.*)"
+ Dataspace:
+ dataspace_id: $id
+ archived_element:
+ type: BooleanElement
+ match_name: "archived"
+ match_value: "(?P<archived>.*)"
records:
- Person:
- full_name_nonlatin: $full_name_nonlatin
- family_name_element:
- type: DictTextElement
- match_name: "family_name"
- match_value: "(?P<family_name>.*)"
+ Dataspace:
+ archived: $archived
+ url_element:
+ type: TextElement
+ match_name: "url"
+ match_value: "(?P<url>.*)"
records:
- Person:
- family_name: $family_name
- given_name_element:
- type: DictTextElement
- match_name: "given_name"
- match_value: "(?P<given_name>.*)"
+ Dataspace:
+ url: $url
+ coordinator_element:
+ type: DictElement
+ match_name: "coordinator"
records:
Person:
- given_name: $given_name
- email_element:
- type: DictTextElement
- match_name: "email"
- match_value: "(?P<email>.*)"
+ parents:
+ - Person
+ Dataspace:
+ Person: $Person
+ subtree: &person_subtree
+ full_name_element:
+ type: TextElement
+ match_name: "full_name"
+ match_value: "(?P<full_name>.*)"
+ records:
+ Person:
+ full_name: $full_name
+ full_name_nonlatin_element:
+ type: TextElement
+ match_name: "full_name_nonlatin"
+ match_value: "(?P<full_name_nonlatin>.*)"
+ records:
+ Person:
+ full_name_nonlatin: $full_name_nonlatin
+ family_name_element:
+ type: TextElement
+ match_name: "family_name"
+ match_value: "(?P<family_name>.*)"
+ records:
+ Person:
+ family_name: $family_name
+ given_name_element:
+ type: TextElement
+ match_name: "given_name"
+ match_value: "(?P<given_name>.*)"
+ records:
+ Person:
+ given_name: $given_name
+ email_element:
+ type: TextElement
+ match_name: "email"
+ match_value: "(?P<email>.*)"
+ records:
+ Person:
+ email: $email
+ affiliation_element:
+ type: TextElement
+ match_name: "affiliation"
+ match_value: "(?P<affiliation>.*)"
+ records:
+ Person:
+ affiliation: $affiliation
+ ORCID_element:
+ type: TextElement
+ match_name: "ORCID"
+ match_value: "(?P<ORCID>.*)"
+ records:
+ Person:
+ ORCID: $ORCID
+ start_date_element:
+ type: TextElement
+ match_name: "start_date"
+ match_value: "(?P<start_date>.*)"
records:
- Person:
- email: $email
- affiliation_element:
- type: DictTextElement
- match_name: "affiliation"
- match_value: "(?P<affiliation>.*)"
+ Dataspace:
+ start_date: $start_date
+ end_date_element:
+ type: TextElement
+ match_name: "end_date"
+ match_value: "(?P<end_date>.*)"
records:
- Person:
- affiliation: $affiliation
- ORCID_element:
- type: DictTextElement
- match_name: "ORCID"
- match_value: "(?P<ORCID>.*)"
+ Dataspace:
+ end_date: $end_date
+ comment:
+ type: TextElement
+ match_name: "comment"
+ match_value: "(?P<comment>.*)"
records:
- Person:
- ORCID: $ORCID
- start_date_element:
- type: DictTextElement
- match_name: "start_date"
- match_value: "(?P<start_date>.*)"
- records:
- Dataspace:
- start_date: $start_date
- end_date_element:
- type: DictTextElement
- match_name: "end_date"
- match_value: "(?P<end_date>.*)"
- records:
- Dataspace:
- end_date: $end_date
- comment:
- type: DictTextElement
- match_name: "comment"
- match_value: "(?P<comment>.*)"
- records:
- Dataspace:
- comment: $comment
+ Dataspace:
+ comment: $comment
raw_data_dir:
type: Directory
match: 03_raw_data
@@ -151,370 +155,374 @@ Data:
match: metadata.json
validate: schema/dataset.schema.json
subtree:
- title_element:
- type: DictTextElement
- match_name: "title"
- match_value: "(?P<title>.*)"
- records:
- Dataset:
- title: $title
- authors_element:
- type: DictListElement
- match_name: "authors"
+ jsondict:
+ type: DictElement
+ match: .*
subtree:
- author_element:
- type: Dict
+ title_element:
+ type: TextElement
+ match_name: "title"
+ match_value: "(?P<title>.*)"
records:
- Person:
- parents:
- - Person
Dataset:
- authors: +$Person
- subtree: *person_subtree
- abstract_element:
- type: DictTextElement
- match_name: "abstract"
- match_value: "(?P<abstract>.*)"
- records:
- Dataset:
- abstract: $abstract
- comment_element:
- type: DictTextElement
- match_name: "comment"
- match_value: "(?P<comment>.*)"
- records:
- Dataset:
- comment: $comment
- license_element:
- type: DictTextElement
- match_name: "license"
- match_value: "(?P<license_name>.*)"
- records:
- license:
- # TODO: As soon as such things can be validated, a
- # creation of a new license has to be forbidden here
- # (although this is effectively done already by
- # validating against the above schema.)
- name: $license_name
- Dataset:
- license: $license
- dataset_doi_element:
- type: DictTextElement
- match_name: "dataset_doi"
- match_value: "(?P<dataset_doi>.*)"
- records:
- Dataset:
- dataset_doi: $dataset_doi
- related_to_dois_element:
- type: DictListElement
- match_name: "related_to_dois"
- subtree:
- related_to_doi_element:
+ title: $title
+ authors_element:
+ type: ListElement
+ match_name: "authors"
+ subtree:
+ author_element:
+ type: DictElement
+ records:
+ Person:
+ parents:
+ - Person
+ Dataset:
+ authors: +$Person
+ subtree: *person_subtree
+ abstract_element:
type: TextElement
- match: "(?P<related_to_doi>).*"
+ match_name: "abstract"
+ match_value: "(?P<abstract>.*)"
records:
Dataset:
- related_to_dois: +$related_to_doi
- Keywords_element:
- type: DictListElement
- match_name: "Keyword"
- Events_element:
- type: DictListElement
- match_name: "Event"
- subtree:
- Event_element:
- type: Dict
+ abstract: $abstract
+ comment_element:
+ type: TextElement
+ match_name: "comment"
+ match_value: "(?P<comment>.*)"
records:
- Event:
- parents:
- - Event
Dataset:
- Event: +$Event
+ comment: $comment
+ license_element:
+ type: TextElement
+ match_name: "license"
+ match_value: "(?P<license_name>.*)"
+ records:
+ license:
+ # TODO: As soon as such things can be validated, a
+ # creation of a new license has to be forbidden here
+ # (although this is effectively done already by
+ # validating against the above schema.)
+ name: $license_name
+ Dataset:
+ license: $license
+ dataset_doi_element:
+ type: TextElement
+ match_name: "dataset_doi"
+ match_value: "(?P<dataset_doi>.*)"
+ records:
+ Dataset:
+ dataset_doi: $dataset_doi
+ related_to_dois_element:
+ type: ListElement
+ match_name: "related_to_dois"
subtree:
- label_element:
- type: DictTextElement
- match_name: "label"
- match_value: "(?P<label>.*)"
- records:
- Event:
- label: $label
- comment_element:
- type: DictTextElement
- match_name: "comment"
- match_value: "(?P<comment>.*)"
+ related_to_doi_element:
+ type: TextElement
+ match_value: "(?P<related_to_doi>).*"
records:
- Event:
- comment: $comment
- start_datetime_element:
- type: DictTextElement
- match_name: start_datetime
- match_value: "(?P<start_datetime>.*)"
- records:
- Event:
- start_datetime: $start_datetime
- end_datetime_element:
- type: DictTextElement
- match_name: end_datetime
- match_value: "(?P<end_datetime>.*)"
- records:
- Event:
- end_datetime: $end_datetime
- longitude_element:
- type: DictFloatElement
- match_name: "longitude"
- match_value: "(?P<longitude>.*)"
- records:
- Event:
- longitude: $longitude
- latitude_element:
- type: DictFloatElement
- match_name: "latitude"
- match_value: "(?P<latitude>.*)"
- records:
- Event:
- latitude: $latitude
- elevation_element:
- type: DictFloatElement
- match_name: "elevation"
- match_value: "(?P<elevation>.*)"
- records:
- Event:
- elevation: $elevation
- location_element:
- type: DictTextElement
- match_name: location
- match_value: "(?P<location>.*)"
- records:
- Event:
- location: $location
- igsn_element:
- type: DictTextElement
- match_name: igsn
- match_value: "(?P<igsn>.*)"
+ Dataset:
+ related_to_dois: +$related_to_doi
+ Keywords_element:
+ type: ListElement
+ match_name: "Keyword"
+ Events_element:
+ type: ListElement
+ match_name: "Event"
+ subtree:
+ Event_element:
+ type: DictElement
records:
Event:
- igsn: $igsn
- events_in_data_element:
- type: DictBooleanElement
- match_name: "events_in_data"
- match_value: "(?P<events_in_data>.*)"
- records:
- Dataset:
- events_in_data: $events_in_data
- geojson_element:
- type: DictTextElement
- match_name: "geojson"
- match_value: "(?P<geojson>.*)"
- records:
- Dataset:
- geojson: $geojson
- project_element:
- type: DictDictElement
- match_name: "project"
- records:
- Project:
- parents:
- - Project
- Dataset:
- Project: $Project
- subtree:
- name_element:
- type: DictTextElement
- match_name: "name"
- match_value: "(?P<name>.*)"
- records:
- Project:
- name: $name
- full_name_element:
- type: DictTextElement
- match_name: "full_name"
- match_value: "(?P<full_name>.*)"
- records:
- Project:
- full_name: $full_name
- project_id_element:
- type: DictTextElement
- match_name: "project_id"
- match_value: "(?P<project_id>.*)"
- records:
- Project:
- project_id: $project_id
- project_type_element:
- type: DictTextElement
- match_name: "project_type"
- match_value: "(?P<project_type_name>.*)"
- records:
- project_type:
- name: $project_type_name
- Project:
- project_type: $project_type
- institute_element:
- type: DictTextElement
- match_name: "institute"
- match_value: "(?P<institute>.*)"
- records:
- Project:
- institute: $institute
- start_date_element:
- type: DictTextElement
- match_name: "start_date"
- match_value: "(?P<start_date>.*)"
+ parents:
+ - Event
+ Dataset:
+ Event: +$Event
+ subtree:
+ label_element:
+ type: TextElement
+ match_name: "label"
+ match_value: "(?P<label>.*)"
+ records:
+ Event:
+ label: $label
+ comment_element:
+ type: TextElement
+ match_name: "comment"
+ match_value: "(?P<comment>.*)"
+ records:
+ Event:
+ comment: $comment
+ start_datetime_element:
+ type: TextElement
+ match_name: start_datetime
+ match_value: "(?P<start_datetime>.*)"
+ records:
+ Event:
+ start_datetime: $start_datetime
+ end_datetime_element:
+ type: TextElement
+ match_name: end_datetime
+ match_value: "(?P<end_datetime>.*)"
+ records:
+ Event:
+ end_datetime: $end_datetime
+ longitude_element:
+ type: FloatElement
+ match_name: "longitude"
+ match_value: "(?P<longitude>.*)"
+ records:
+ Event:
+ longitude: $longitude
+ latitude_element:
+ type: FloatElement
+ match_name: "latitude"
+ match_value: "(?P<latitude>.*)"
+ records:
+ Event:
+ latitude: $latitude
+ elevation_element:
+ type: FloatElement
+ match_name: "elevation"
+ match_value: "(?P<elevation>.*)"
+ records:
+ Event:
+ elevation: $elevation
+ location_element:
+ type: TextElement
+ match_name: location
+ match_value: "(?P<location>.*)"
+ records:
+ Event:
+ location: $location
+ igsn_element:
+ type: TextElement
+ match_name: igsn
+ match_value: "(?P<igsn>.*)"
+ records:
+ Event:
+ igsn: $igsn
+ events_in_data_element:
+ type: BooleanElement
+ match_name: "events_in_data"
+ match_value: "(?P<events_in_data>.*)"
records:
- Project:
- start_date: $start_date
- end_date_element:
- type: DictTextElement
- match_name: "end_date"
- match_value: "(?P<end_date>.*)"
+ Dataset:
+ events_in_data: $events_in_data
+ geojson_element:
+ type: TextElement
+ match_name: "geojson"
+ match_value: "(?P<geojson>.*)"
records:
- Project:
- end_date: $end_date
- url_element:
- type: DictTextElement
- match_name: "url"
- match_value: "(?P<url>.*)"
+ Dataset:
+ geojson: $geojson
+ project_element:
+ type: DictElement
+ match_name: "project"
records:
Project:
- url: $url
- coordinators_element:
- type: DictListElement
- match_name: "coordinators"
- subtree:
- coordinator_element:
- type: Dict
- records:
- Person:
- parents:
- - Person
- Project:
- coordinators: +$Person
- subtree: *person_subtree
- campaign_element:
- type: DictDictElement
- match_name: "campaign"
- records:
- Campaign:
parents:
- - Campaign
+ - Project
Dataset:
- Campaign: $Campaign
+ Project: $Project
subtree:
- label_element:
- type: DictTextElement
- match_name: "label"
- match_value: "(?P<label>.*)"
+ name_element:
+ type: TextElement
+ match_name: "name"
+ match_value: "(?P<name>.*)"
records:
- Campaign:
- label: $label
- optional_label_element:
- type: DictTextElement
- match_name: "optional_label"
- match_value: "(?P<optional_label>.*)"
+ Project:
+ name: $name
+ full_name_element:
+ type: TextElement
+ match_name: "full_name"
+ match_value: "(?P<full_name>.*)"
records:
- Campaign:
- optional_label: $optional_label
+ Project:
+ full_name: $full_name
+ project_id_element:
+ type: TextElement
+ match_name: "project_id"
+ match_value: "(?P<project_id>.*)"
+ records:
+ Project:
+ project_id: $project_id
+ project_type_element:
+ type: TextElement
+ match_name: "project_type"
+ match_value: "(?P<project_type_name>.*)"
+ records:
+ project_type:
+ name: $project_type_name
+ Project:
+ project_type: $project_type
+ institute_element:
+ type: TextElement
+ match_name: "institute"
+ match_value: "(?P<institute>.*)"
+ records:
+ Project:
+ institute: $institute
start_date_element:
- type: DictTextElement
+ type: TextElement
match_name: "start_date"
match_value: "(?P<start_date>.*)"
records:
- Campaign:
+ Project:
start_date: $start_date
end_date_element:
- type: DictTextElement
+ type: TextElement
match_name: "end_date"
match_value: "(?P<end_date>.*)"
records:
- Campaign:
+ Project:
end_date: $end_date
- responsible_scientists_element:
- type: DictListElement
- match_name: "responsible_scientists"
+ url_element:
+ type: TextElement
+ match_name: "url"
+ match_value: "(?P<url>.*)"
+ records:
+ Project:
+ url: $url
+ coordinators_element:
+ type: ListElement
+ match_name: "coordinators"
subtree:
- responsible_scientist_element:
- type: Dict
+ coordinator_element:
+ type: DictElement
records:
Person:
parents:
- Person
- Campaign:
- responsible_scientists: +$Person
+ Project:
+ coordinators: +$Person
subtree: *person_subtree
- Methods_element:
- type: DictListElement
- match_name: "Method"
- subtree:
- Method_element:
- type: Dict
+ campaign_element:
+ type: DictElement
+ match_name: "campaign"
records:
- Method:
+ Campaign:
parents:
- - Method
+ - Campaign
Dataset:
- Method: +$Method
+ Campaign: $Campaign
subtree:
- method_name_element:
- type: DictTextElement
- match_name: "method_name"
- match_value: "(?P<method_name>.*)"
+ label_element:
+ type: TextElement
+ match_name: "label"
+ match_value: "(?P<label>.*)"
records:
- Method:
- name: $method_name
- abbreviation_element:
- type: DictTextElement
- match_name: "abbreviation"
- match_value: "(?P<abbreviation>.*)"
+ Campaign:
+ label: $label
+ optional_label_element:
+ type: TextElement
+ match_name: "optional_label"
+ match_value: "(?P<optional_label>.*)"
records:
- Method:
- abbreviation: $abbreviation
- url_element:
- type: DictTextElement
- match_name: "url"
- match_value: "(?P<url>.*)"
+ Campaign:
+ optional_label: $optional_label
+ start_date_element:
+ type: TextElement
+ match_name: "start_date"
+ match_value: "(?P<start_date>.*)"
+ records:
+ Campaign:
+ start_date: $start_date
+ end_date_element:
+ type: TextElement
+ match_name: "end_date"
+ match_value: "(?P<end_date>.*)"
+ records:
+ Campaign:
+ end_date: $end_date
+ responsible_scientists_element:
+ type: ListElement
+ match_name: "responsible_scientists"
+ subtree:
+ responsible_scientist_element:
+ type: DictElement
+ records:
+ Person:
+ parents:
+ - Person
+ Campaign:
+ responsible_scientists: +$Person
+ subtree: *person_subtree
+ Methods_element:
+ type: ListElement
+ match_name: "Method"
+ subtree:
+ Method_element:
+ type: DictElement
records:
Method:
- url: $url
- Taxa_element:
- type: DictListElement
- match_name: "Taxon"
- subtree:
- Taxon_element:
- type: Dict
- records:
- Taxon:
- parents:
- - Taxon
- Dataset:
- Taxon: +$Taxon
+ parents:
+ - Method
+ Dataset:
+ Method: +$Method
+ subtree:
+ method_name_element:
+ type: TextElement
+ match_name: "method_name"
+ match_value: "(?P<method_name>.*)"
+ records:
+ Method:
+ name: $method_name
+ abbreviation_element:
+ type: TextElement
+ match_name: "abbreviation"
+ match_value: "(?P<abbreviation>.*)"
+ records:
+ Method:
+ abbreviation: $abbreviation
+ url_element:
+ type: TextElement
+ match_name: "url"
+ match_value: "(?P<url>.*)"
+ records:
+ Method:
+ url: $url
+ Taxa_element:
+ type: ListElement
+ match_name: "Taxon"
subtree:
- taxon_name_element:
- type: DictTextElement
- match_name: "taxon_name"
- match_value: "(?P<taxon_name>.*)"
+ Taxon_element:
+ type: DictElement
records:
Taxon:
- name: $taxon_name
- archived_element:
- type: DictBooleanElement
- match_name: "archived"
- match_value: "(P<archived>.*)"
- records:
- Dataset:
- archived: $archived
- publication_date_element:
- type: DictTextElement
- match_name: "publication_date"
- match_value: "(P<publication_date>.*)"
- records:
- Dataset:
- publication_date: $publication_date
- max_files_element:
- type: DictIntegerElement
- match_name: "max_files"
- match_value: "(P<max_files>.*)"
- records:
- Dataset:
- max_files: $max_files
+ parents:
+ - Taxon
+ Dataset:
+ Taxon: +$Taxon
+ subtree:
+ taxon_name_element:
+ type: TextElement
+ match_name: "taxon_name"
+ match_value: "(?P<taxon_name>.*)"
+ records:
+ Taxon:
+ name: $taxon_name
+ archived_element:
+ type: BooleanElement
+ match_name: "archived"
+ match_value: "(P<archived>.*)"
+ records:
+ Dataset:
+ archived: $archived
+ publication_date_element:
+ type: TextElement
+ match_name: "publication_date"
+ match_value: "(P<publication_date>.*)"
+ records:
+ Dataset:
+ publication_date: $publication_date
+ max_files_element:
+ type: IntegerElement
+ match_name: "max_files"
+ match_value: "(P<max_files>.*)"
+ records:
+ Dataset:
+ max_files: $max_files
auxiliary_file: &aux_file_template
type: File
match: "(?P<aux_file_name>(?!metadata.json).*)"
diff --git a/integrationtests/test_issues.py b/integrationtests/test_issues.py
index 0f8934137e646677243a851f8c525d90375fb66d..527b4c0cf67f483d5b61972a0104ff4fb673402d 100644
--- a/integrationtests/test_issues.py
+++ b/integrationtests/test_issues.py
@@ -22,7 +22,7 @@ import caosdb as db
from caoscrawler.crawl import Crawler
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
-from caoscrawler.structure_elements import Dict
+from caoscrawler.structure_elements import DictElement
from caosdb.utils.register_tests import clear_database, set_test_key
set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
@@ -51,14 +51,14 @@ def test_issue_23(clear_database):
# identifying_prop and prop_b, but not prop_a ...
crawler_definition = {
"DictTest": {
- "type": "Dict",
+ "type": "DictElement",
"match": "(.*)",
"records": {
"TestType": {}
},
"subtree": {
"identifying_element": {
- "type": "DictTextElement",
+ "type": "TextElement",
"match_name": "ident",
"match_value": "(?P<ident_value>.*)",
"records": {
@@ -68,7 +68,7 @@ def test_issue_23(clear_database):
}
},
"other_element": {
- "type": "DictTextElement",
+ "type": "TextElement",
"match_name": "prop_b",
"match_value": "(?P<other_value>.*)",
"records": {
@@ -96,7 +96,7 @@ def test_issue_23(clear_database):
}
records = crawler.start_crawling(
- Dict("TestDict", test_dict), crawler_definition, converter_registry)
+ DictElement("TestDict", test_dict), crawler_definition, converter_registry)
assert len(records) == 1
rec_crawled = records[0]
diff --git a/integrationtests/test_realworld_example.py b/integrationtests/test_realworld_example.py
index 48729f720e5b11eb5ad9722653aea06756cb0ae8..4158ed22278ef5c871a22d45885e58fbfa84ea3b 100644
--- a/integrationtests/test_realworld_example.py
+++ b/integrationtests/test_realworld_example.py
@@ -31,9 +31,8 @@ import os
import caosdb as db
from caoscrawler.crawl import Crawler, crawler_main
-from caoscrawler.converters import JSONFileConverter, DictConverter
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
-from caoscrawler.structure_elements import File, JSONFile, Directory
+from caoscrawler.structure_elements import Directory
import pytest
from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index f818888e98690a861228b1f3c0214b1cc94fb6e1..0000000000000000000000000000000000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-testpaths=unittests
diff --git a/setup.cfg b/setup.cfg
index 810b5e9262b2a2809aa6a54b6d67665b36ff6526..e16a49cbbb55699db9abd37fbc5890eca5634ef6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = caoscrawler
-version = 0.2.0
+version = 0.3.0
author = Alexander Schlemmer
author_email = alexander.schlemmer@ds.mpg.de
description = A new crawler for caosdb
@@ -17,15 +17,16 @@ classifiers =
package_dir =
= src
packages = find:
-python_requires = >=3.8
+python_requires = >=3.7
install_requires =
importlib-resources
- caosdb > 0.9.0
+ caosdb >= 0.11.0
caosadvancedtools >= 0.6.0
yaml-header-tools >= 0.2.1
pyyaml
odfpy #make optional
pandas
+ importlib_metadata;python_version<'3.8'
[options.packages.find]
where = src
diff --git a/src/caoscrawler/__init__.py b/src/caoscrawler/__init__.py
index b65b9fd9d24b9519a52ca13d07e46c9d8f791a73..044d8f0bf53c4c80dab9b492919fa64ab321a60d 100644
--- a/src/caoscrawler/__init__.py
+++ b/src/caoscrawler/__init__.py
@@ -1 +1,2 @@
from .crawl import Crawler, SecurityMode
+from .version import CfoodRequiredVersionError, version as __version__
diff --git a/src/caoscrawler/cfood-schema.yml b/src/caoscrawler/cfood-schema.yml
index d7b5abfd1ac6c381b50bd4ce61015f1b8602b408..5e724c83695e098ce980e1aa8e81c65ae8525e19 100644
--- a/src/caoscrawler/cfood-schema.yml
+++ b/src/caoscrawler/cfood-schema.yml
@@ -16,10 +16,15 @@ cfood:
- YamlFileCaosDBRecord
- MarkdownFile
- DictListElement
+ - ListElement
- DictDictElement
+ - DictElement
- DictFloatElement
+ - FloatElement
- DictIntegerElement
+ - IntegerElement
- DictBooleanElement
+ - BooleanElement
- Definitions
- Dict
- JSONFile
diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py
index def7ca1243156b2d25a1f8db387fef25e1cc859c..d4e25f73a8a9e7dad42c50d907745dfb7329bb13 100644
--- a/src/caoscrawler/converters.py
+++ b/src/caoscrawler/converters.py
@@ -23,19 +23,21 @@
# ** end header
#
+from __future__ import annotations
from jsonschema import validate, ValidationError
+
import os
import re
+import datetime
import caosdb as db
import json
import warnings
from .utils import has_parent
from .stores import GeneralStore, RecordStore
-from .structure_elements import (StructureElement, Directory, File, Dict, JSONFile,
- DictIntegerElement, DictBooleanElement,
- DictFloatElement, DictDictElement,
- TextElement, DictTextElement, DictElement, DictListElement)
-from typing import Dict as Dict_t, List, Optional, Tuple, Union
+from .structure_elements import (StructureElement, Directory, File, DictElement, JSONFile,
+ IntegerElement, BooleanElement, FloatElement, NoneElement,
+ TextElement, TextElement, ListElement)
+from typing import List, Optional, Tuple, Union
from abc import ABCMeta, abstractmethod
from string import Template
import yaml_header_tools
@@ -75,8 +77,29 @@ def str_to_bool(x):
else:
raise RuntimeError("Should be 'true' or 'false'.")
+# TODO: Comment on types and inheritance
+# Currently, we often check the type of StructureElements, because serveral converters assume that
+# they are called only with the appropriate class.
+# Raising an Error if the type is not sufficient (e.g. TextElement instead of DictElement) means
+# that the generic parent class StructureElement is actually NOT a valid type for the argument and
+# type hints should reflect this.
+# However, we should not narrow down the type of the arguments compared to the function definitions
+# in the parent Converter class. See
+# - https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides
+# - https://stackoverflow.com/questions/56860/what-is-an-example-of-the-liskov-substitution-principle
+# - https://blog.daftcode.pl/covariance-contravariance-and-invariance-the-ultimate-python-guide-8fabc0c24278
+# Thus, the problem lies in the following design:
+# Converter instances are supposed to be used by the Crawler in a generic way (The crawler calls
+# `match` and `typecheck` etc) but the functions are not supposed to be called with generic
+# StructureElements. One direction out of this would be a refactoring that makes the crawler class
+# expose a generic function like `treat_element`, which can be called with any StructureElement and
+# the Converter decides what to do (e.g. do nothing if the type is one that it does not care
+# about).
+# https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/64
+
class ConverterValidationError(Exception):
+
"""To be raised if contents of an element to be converted are invalid."""
def __init__(self, msg):
@@ -176,9 +199,7 @@ def handle_value(value: Union[dict, str, list], values: GeneralStore):
return (propvalue, collection_mode)
-def create_records(values: GeneralStore,
- records: RecordStore,
- def_records: dict):
+def create_records(values: GeneralStore, records: RecordStore, def_records: dict):
# list of keys to identify, which variables have been set by which paths:
# the items are tuples:
# 0: record name
@@ -264,14 +285,12 @@ class Converter(object, metaclass=ABCMeta):
Converters treat StructureElements contained in the hierarchical sturcture.
"""
- def __init__(self, definition: dict,
- name: str,
- converter_registry: dict):
+ def __init__(self, definition: dict, name: str, converter_registry: dict):
self.definition = definition
self.name = name
# Used to store usage information for debugging:
- self.metadata: Dict_t[str, set[str]] = {
+ self.metadata: dict[str, set[str]] = {
"usage": set()
}
@@ -284,9 +303,7 @@ class Converter(object, metaclass=ABCMeta):
converter_definition, converter_name, converter_registry))
@staticmethod
- def converter_factory(definition: dict,
- name: str,
- converter_registry: dict):
+ def converter_factory(definition: dict, name: str, converter_registry: dict):
"""creates a Converter instance of the appropriate class.
The `type` key in the `definition` defines the Converter class which is being used.
@@ -363,7 +380,7 @@ class Converter(object, metaclass=ABCMeta):
filtered_children = FILTER_FUNCTIONS[rule](to_be_filtered)
- return filtered_children+unmatched_children
+ return filtered_children + unmatched_children
@abstractmethod
def typecheck(self, element: StructureElement):
@@ -373,6 +390,58 @@ class Converter(object, metaclass=ABCMeta):
"""
pass
+ @staticmethod
+ def _debug_matching_template(name: str, regexp: list[str], matched: list[str], result: Optional[dict]):
+ """ Template for the debugging output for the match function """
+ print("\n--------", name, "-----------")
+ for re, ma in zip(regexp, matched):
+ print("matching against:\n" + re)
+ print("matching:\n" + ma)
+ print("---------")
+ if result is None:
+ print("No match")
+ else:
+ print("Matched groups:")
+ print(result)
+ print("----------------------------------------")
+
+ @staticmethod
+ def debug_matching(kind=None):
+ def debug_matching_decorator(func):
+ """
+ decorator for the match function of Converters that implements debug for the match of
+ StructureElements
+ """
+
+ def inner(self, element: StructureElement):
+ mr = func(self, element)
+ if "debug_match" in self.definition and self.definition["debug_match"]:
+ if kind == "name" and "match" in self.definition:
+ self._debug_matching_template(name=self.__class__.__name__,
+ regexp=[self.definition["match"]],
+ matched=[element.name],
+ result=mr)
+ elif kind == "name_and_value":
+ self._debug_matching_template(
+ name=self.__class__.__name__,
+ regexp=[self.definition["match"]
+ if "match" in self.definition else "",
+ self.definition["match_name"]
+ if "match_name" in self.definition else "",
+ self.definition["match_value"]
+ if "match_value" in self.definition else ""],
+ matched=[element.name, element.name, str(element.value)],
+ result=mr)
+ else:
+ self._debug_matching_template(name=self.__class__.__name__,
+ regexp=self.definition["match"]
+ if "match" in self.definition else "",
+ matched=str(element),
+ result=mr)
+ return mr
+ return inner
+ return debug_matching_decorator
+
@abstractmethod
def match(self, element: StructureElement) -> Optional[dict]:
"""
@@ -386,16 +455,8 @@ class Converter(object, metaclass=ABCMeta):
class DirectoryConverter(Converter):
-
- def __init__(self, definition: dict, name: str,
- converter_registry: dict):
- """
- Initialize a new directory converter.
- """
- super().__init__(definition, name, converter_registry)
-
- def create_children(self, generalStore: GeneralStore,
- element: StructureElement):
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, Directory):
raise RuntimeError(
"Directory converters can only create children from directories.")
@@ -413,7 +474,11 @@ class DirectoryConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, Directory)
+ # TODO basically all converters implement such a match function. Shouldn't this be the one
+ # of the parent class and subclasses can overwrite if needed?
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, Directory):
raise RuntimeError("Element must be a directory.")
m = re.match(self.definition["match"], element.name)
@@ -450,11 +515,12 @@ class SimpleFileConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, File)
- def create_children(self, generalStore: GeneralStore,
- element: StructureElement):
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
return list()
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("Element must be a file.")
m = re.match(self.definition["match"], element.name)
@@ -463,16 +529,20 @@ class SimpleFileConverter(Converter):
return m.groupdict()
+class FileConverter(SimpleFileConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use SimpleFileConverter."))
+ super().__init__(*args, **kwargs)
+
+
class MarkdownFileConverter(Converter):
- def __init__(self, definition: dict, name: str,
- converter_registry: dict):
- """
- Initialize a new directory converter.
- """
- super().__init__(definition, name, converter_registry)
+ """
+ reads the yaml header of markdown files (if a such a header exists).
+ """
- def create_children(self, generalStore: GeneralStore,
- element: StructureElement):
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("A markdown file is needed to create children.")
@@ -482,9 +552,9 @@ class MarkdownFileConverter(Converter):
for name, entry in header.items():
if type(entry) == list:
- children.append(DictListElement(name, entry))
+ children.append(ListElement(name, entry))
elif type(entry) == str:
- children.append(DictTextElement(name, entry))
+ children.append(TextElement(name, entry))
else:
raise RuntimeError(
"Header entry {} has incompatible type.".format(name))
@@ -493,7 +563,9 @@ class MarkdownFileConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, File)
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("Element must be a file.")
m = re.match(self.definition["match"], element.name)
@@ -508,58 +580,118 @@ class MarkdownFileConverter(Converter):
return m.groupdict()
-class DictConverter(Converter):
- # TODO use Dict as typecheck?
+def convert_basic_element(element: Union[list, dict, bool, int, float, str, None], name=None,
+ msg_prefix=""):
+ """converts basic Python objects to the corresponding StructureElements """
+ if isinstance(element, list):
+ return ListElement(name, element)
+ elif isinstance(element, dict):
+ return DictElement(name, element)
+ elif isinstance(element, bool):
+ return BooleanElement(name, element)
+ elif isinstance(element, int):
+ return IntegerElement(name, element)
+ elif isinstance(element, float):
+ return FloatElement(name, element)
+ elif isinstance(element, str):
+ return TextElement(name, element)
+ elif element is None:
+ return NoneElement(name)
+ elif isinstance(element, datetime.date):
+ return TextElement(name, str(element))
+ else:
+ raise NotImplementedError(
+ msg_prefix + f"The object that has an unexpected type: {type(element)}\n"
+ f"The object is:\n{str(element)}")
+
+
+def validate_against_json_schema(instance, schema_resource: Union[dict, str]):
+ """validates given ``instance`` against given ``schema_resource``.
+
+ Args:
+ instance: instance to be validated, typically ``dict`` but can be ``list``, ``str``, etc.
+ schema_resource: Either a path to the JSON file containing the schema or a ``dict`` with
+ the schema
+ """
+ if isinstance(schema_resource, dict):
+ schema = schema_resource
+ elif isinstance(schema_resource, str):
+ with open(schema_resource, 'r') as json_file:
+ schema = json.load(json_file)
+ else:
+ raise ValueError("The value of 'validate' has to be a string describing the path "
+ "to the json schema file (relative to the cfood yml) "
+ "or a dict containing the schema.")
+ # validate instance (e.g. JSON content) against schema
+ try:
+ validate(instance=instance, schema=schema)
+ except ValidationError as err:
+ raise ConverterValidationError(
+ f"\nCouldn't validate {instance}:\n{err.message}")
+
+
+class DictElementConverter(Converter):
def create_children(self, generalStore: GeneralStore, element: StructureElement):
- if not self.typecheck(element):
- raise RuntimeError("A dict is needed to create children")
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, DictElement):
+ raise ValueError("create_children was called with wrong type of StructureElement")
- return self._create_children_from_dict(element.value)
+ try:
+ return self._create_children_from_dict(element.value)
+ except ConverterValidationError as err:
+ path = generalStore[self.name]
+ raise ConverterValidationError(
+ "Error during the validation of the dictionary located at the following node "
+ "in the data structure:\n"
+ f"{path}\n" + err.message)
def _create_children_from_dict(self, data):
+ if "validate" in self.definition and self.definition["validate"]:
+ validate_against_json_schema(data, self.definition["validate"])
+
children = []
for name, value in data.items():
- if type(value) == list:
- children.append(DictListElement(name, value))
- elif type(value) == str:
- children.append(DictTextElement(name, value))
- elif type(value) == dict:
- children.append(DictDictElement(name, value))
- elif type(value) == int:
- children.append(DictIntegerElement(name, value))
- elif type(value) == bool:
- children.append(DictBooleanElement(name, value))
- elif type(value) == float:
- children.append(DictFloatElement(name, value))
- elif type(value) == type(None):
- continue
- else:
- children.append(DictElement(name, value))
- warnings.warn(f"The value in the dict for key:{name} has an unknown type. "
- "The fallback type DictElement is used.")
+ children.append(convert_basic_element(
+ value, name, f"The value in the dict for key:{name} has an unknown type."))
return children
- # TODO use Dict as typecheck?
def typecheck(self, element: StructureElement):
- return isinstance(element, Dict)
+ return isinstance(element, DictElement)
+ @Converter.debug_matching("name_and_value")
def match(self, element: StructureElement):
"""
Allways matches if the element has the right type.
"""
- if not isinstance(element, Dict):
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, DictElement):
raise RuntimeError("Element must be a DictElement.")
- return {}
+ return match_name_and_value(self.definition, element.name, element.value)
+
+class DictConverter(DictElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use DictConverter."))
+ super().__init__(*args, **kwargs)
-# TODO: difference to SimpleFileConverter? Do we need both?
-class FileConverter(Converter):
+
+class DictDictElementConverter(DictElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use DictElementConverter."))
+ super().__init__(*args, **kwargs)
+
+
+class JSONFileConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, File)
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not self.typecheck(element):
raise RuntimeError("Element must be a file")
m = re.match(self.definition["match"], element.name)
@@ -568,14 +700,33 @@ class FileConverter(Converter):
return m.groupdict()
def create_children(self, generalStore: GeneralStore, element: StructureElement):
- return []
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, File):
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ with open(element.path, 'r') as json_file:
+ json_data = json.load(json_file)
+ if "validate" in self.definition and self.definition["validate"]:
+ try:
+ validate_against_json_schema(json_data, self.definition["validate"])
+ except ConverterValidationError as err:
+ raise ConverterValidationError(
+ "Error during the validation of the JSON file:\n"
+ f"{element.path}\n" + err.message)
+ structure_element = convert_basic_element(
+ json_data,
+ name=element.name+"_child_dict",
+ msg_prefix="The JSON File contained content that was parsed to a Python object"
+ " with an unexpected type.")
+ return [structure_element]
-class JSONFileConverter(DictConverter):
+class YAMLFileConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, File)
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not self.typecheck(element):
raise RuntimeError("Element must be a file")
m = re.match(self.definition["match"], element.name)
@@ -584,36 +735,78 @@ class JSONFileConverter(DictConverter):
return m.groupdict()
def create_children(self, generalStore: GeneralStore, element: StructureElement):
- if not self.typecheck(element):
- raise RuntimeError("A JSON file is needed to create children")
- # TODO: either add explicit type check for File structure element here,
- # or add a comment to suppress mypy type warning.
- with open(element.path, 'r') as json_file:
- json_data = json.load(json_file)
- if not isinstance(json_data, dict):
- raise NotImplementedError("JSON file must contain a dict")
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, File):
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ with open(element.path, 'r') as yaml_file:
+ yaml_data = yaml.safe_load(yaml_file)
if "validate" in self.definition and self.definition["validate"]:
- if isinstance(self.definition["validate"], dict):
- schema = self.definition["validate"]
- elif isinstance(self.definition["validate"], str):
-
- with open(self.definition["validate"], 'r') as json_file:
- schema = json.load(json_file)
- else:
- raise ValueError("The value of 'validate' has to be a string describing the path "
- "to the json schema file (relative to the cfood yml) "
- "or a dict containing the schema.")
- # Validate the json content
try:
- validate(instance=json_data, schema=schema)
- except ValidationError as err:
+ validate_against_json_schema(yaml_data, self.definition["validate"])
+ except ConverterValidationError as err:
raise ConverterValidationError(
- f"Couldn't validate {json_data}:\n{err.message}")
+ "Error during the validation of the YAML file:\n"
+ f"{element.path}\n" + err.message)
+ structure_element = convert_basic_element(
+ yaml_data,
+ name=element.name+"_child_dict",
+ msg_prefix="The YAML File contained content that was parsed to a Python object"
+ " with an unexpected type.")
+ return [structure_element]
+
+
+def match_name_and_value(definition, name, value):
+ """
+ takes match definitions from the definition argument and applies regular expressiion to name
+ and possibly value
+
+ one of the keys 'match_name' and "match' needs to be available in definition
+ 'match_value' is optional
+
+ Returns None, if match_name or match lead to no match. Otherwise, returns a dictionary with the
+ matched groups, possibly including matches from using match_value
+ """
+ if "match_name" in definition:
+ if "match" in definition:
+ raise RuntimeError(f"Do not supply both, 'match_name' and 'match'.")
+
+ m1 = re.match(definition["match_name"], name)
+ if m1 is None:
+ return None
+ else:
+ m1 = m1.groupdict()
+ elif "match" in definition:
+ m1 = re.match(definition["match"], name)
+ if m1 is None:
+ return None
+ else:
+ m1 = m1.groupdict()
+ else:
+ m1 = {}
- return self._create_children_from_dict(json_data)
+ if "match_value" in definition:
+ m2 = re.match(definition["match_value"], str(value), re.DOTALL)
+ if m2 is None:
+ return None
+ else:
+ m2 = m2.groupdict()
+ else:
+ m2 = {}
+
+ values = dict()
+ values.update(m1)
+ values.update(m2)
+ return values
+
+
+class _AbstractScalarValueElementConverter(Converter):
+ """
+ A base class for all converters that have a scalar value that can be matched using a regular
+ expression.
+ values must have one of the following type: str, bool, int, float
+ """
-class _AbstractDictElementConverter(Converter):
default_matches = {
"accept_text": False,
"accept_bool": False,
@@ -625,8 +818,14 @@ class _AbstractDictElementConverter(Converter):
return []
def typecheck(self, element: StructureElement):
- return True
+ """
+ returns whether the type of StructureElement is accepted by this converter instance.
+ """
+ allowed_matches = self._merge_match_definition_with_default(self.default_matches,
+ self.definition)
+ return self._typecheck(element, allowed_matches)
+ @Converter.debug_matching("name_and_value")
def match(self, element: StructureElement):
"""
Try to match the given structure element.
@@ -636,21 +835,15 @@ class _AbstractDictElementConverter(Converter):
Else return a dictionary containing the variables from the matched regexp
as key value pairs.
"""
- if not self.typecheck(element):
- raise RuntimeError(
- f"Element has an invalid type: {type(element)}.")
- m1 = re.match(self.definition["match_name"], element.name)
- if m1 is None:
- return None
- m2 = re.match(self.definition["match_value"], str(element.value), re.DOTALL)
- if m2 is None:
- return None
- values = dict()
- values.update(m1.groupdict())
- values.update(m2.groupdict())
- return values
-
- def _typecheck(self, element: StructureElement, allowed_matches: Dict):
+ # TODO: See comment on types and inheritance
+ if (not isinstance(element, TextElement)
+ and not isinstance(element, BooleanElement)
+ and not isinstance(element, IntegerElement)
+ and not isinstance(element, FloatElement)):
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ return match_name_and_value(self.definition, element.name, element.value)
+
+ def _typecheck(self, element: StructureElement, allowed_matches: dict):
"""
returns whether the type of StructureElement is accepted.
@@ -661,18 +854,18 @@ class _AbstractDictElementConverter(Converter):
returns: whether or not the converter allows the type of element
"""
- if (bool(allowed_matches["accept_text"]) and isinstance(element, DictTextElement)):
+ if (bool(allowed_matches["accept_text"]) and isinstance(element, TextElement)):
return True
- elif (bool(allowed_matches["accept_bool"]) and isinstance(element, DictBooleanElement)):
+ elif (bool(allowed_matches["accept_bool"]) and isinstance(element, BooleanElement)):
return True
- elif (bool(allowed_matches["accept_int"]) and isinstance(element, DictIntegerElement)):
+ elif (bool(allowed_matches["accept_int"]) and isinstance(element, IntegerElement)):
return True
- elif (bool(allowed_matches["accept_float"]) and isinstance(element, DictFloatElement)):
+ elif (bool(allowed_matches["accept_float"]) and isinstance(element, FloatElement)):
return True
else:
return False
- def _merge_match_definition_with_default(self, default: Dict, definition: Dict):
+ def _merge_match_definition_with_default(self, default: dict, definition: dict):
"""
returns a dict with the same keys as default dict but with updated values from definition
where it has the same keys
@@ -686,16 +879,8 @@ class _AbstractDictElementConverter(Converter):
result[key] = default[key]
return result
- def typecheck(self, element: StructureElement):
- """
- returns whether the type of StructureElement is accepted by this converter instance.
- """
- allowed_matches = self._merge_match_definition_with_default(self.default_matches,
- self.definition)
- return self._typecheck(element, allowed_matches)
-
-class DictBooleanElementConverter(_AbstractDictElementConverter):
+class BooleanElementConverter(_AbstractScalarValueElementConverter):
default_matches = {
"accept_text": False,
"accept_bool": True,
@@ -704,7 +889,14 @@ class DictBooleanElementConverter(_AbstractDictElementConverter):
}
-class DictFloatElementConverter(_AbstractDictElementConverter):
+class DictBooleanElementConverter(BooleanElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use BooleanElementConverter."))
+ super().__init__(*args, **kwargs)
+
+
+class FloatElementConverter(_AbstractScalarValueElementConverter):
default_matches = {
"accept_text": False,
"accept_bool": False,
@@ -713,7 +905,14 @@ class DictFloatElementConverter(_AbstractDictElementConverter):
}
-class DictTextElementConverter(_AbstractDictElementConverter):
+class DictFloatElementConverter(FloatElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use FloatElementConverter."))
+ super().__init__(*args, **kwargs)
+
+
+class TextElementConverter(_AbstractScalarValueElementConverter):
default_matches = {
"accept_text": True,
"accept_bool": True,
@@ -721,8 +920,24 @@ class DictTextElementConverter(_AbstractDictElementConverter):
"accept_float": True,
}
+ def __init__(self, definition, *args, **kwargs):
+ if "match" in definition:
+ raise ValueError("""
+The 'match' key will in future be used to match a potential name of a TextElement. Please use
+the 'match_value' key to match the value of the TextElement and 'match_name' for matching the name.
+""")
+
+ super().__init__(definition, *args, **kwargs)
+
+
+class DictTextElementConverter(TextElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use TextElementConverter."))
+ super().__init__(*args, **kwargs)
+
-class DictIntegerElementConverter(_AbstractDictElementConverter):
+class IntegerElementConverter(_AbstractScalarValueElementConverter):
default_matches = {
"accept_text": False,
"accept_bool": False,
@@ -731,76 +946,56 @@ class DictIntegerElementConverter(_AbstractDictElementConverter):
}
-class DictListElementConverter(Converter):
+class DictIntegerElementConverter(IntegerElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use IntegerElementConverter."))
+ super().__init__(*args, **kwargs)
+
+
+class ListElementConverter(Converter):
def create_children(self, generalStore: GeneralStore,
element: StructureElement):
- if not isinstance(element, DictListElement):
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, ListElement):
raise RuntimeError(
"This converter can only process DictListElements.")
- children = []
+ children: list[StructureElement] = []
for index, list_element in enumerate(element.value):
# TODO(fspreck): Refactor this and merge with DictXXXElements maybe?
if isinstance(list_element, str):
children.append(TextElement(str(index), list_element))
elif isinstance(list_element, dict):
- children.append(Dict(str(index), list_element))
+ children.append(DictElement(str(index), list_element))
+ elif isinstance(list_element, StructureElement):
+ children.append(list_element)
else:
raise NotImplementedError(
f"Unkown type {type(list_element)} in list element {list_element}.")
return children
def typecheck(self, element: StructureElement):
- return isinstance(element, DictListElement)
-
- def match(self, element: StructureElement):
- if not isinstance(element, DictListElement):
- raise RuntimeError("Element must be a DictListElement.")
- m = re.match(self.definition["match_name"], element.name)
- if m is None:
- return None
- if "match" in self.definition:
- raise NotImplementedError(
- "Match is not implemented for DictListElement.")
- return m.groupdict()
-
-
-class DictDictElementConverter(DictConverter):
- def create_children(self, generalStore: GeneralStore, element: StructureElement):
- if not self.typecheck(element):
- raise RuntimeError("A dict is needed to create children")
-
- return self._create_children_from_dict(element.value)
-
- def typecheck(self, element: StructureElement):
- return isinstance(element, DictDictElement)
+ return isinstance(element, ListElement)
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
- if not self.typecheck(element):
- raise RuntimeError("Element must be a DictDictElement.")
+ # TODO: See comment on types and inheritance
+ if not isinstance(element, ListElement):
+ raise RuntimeError("Element must be a ListElement.")
m = re.match(self.definition["match_name"], element.name)
if m is None:
return None
if "match" in self.definition:
raise NotImplementedError(
- "Match is not implemented for DictDictElement.")
+ "Match is not implemented for ListElement.")
return m.groupdict()
-class TextElementConverter(Converter):
- def create_children(self, generalStore: GeneralStore,
- element: StructureElement):
- return []
-
- def typecheck(self, element: StructureElement):
- return isinstance(element, TextElement)
-
- def match(self, element: StructureElement):
- if not isinstance(element, TextElement):
- raise RuntimeError("Element must be a TextElement.")
- m = re.match(self.definition["match"], element.value)
- if m is None:
- return None
- return m.groupdict()
+class DictListElementConverter(ListElementConverter):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning(
+ "This class is depricated. Please use ListElementConverter."))
+ super().__init__(*args, **kwargs)
class TableConverter(Converter):
@@ -808,10 +1003,10 @@ class TableConverter(Converter):
This converter reads tables in different formats line by line and
allows matching the corresponding rows.
- The subtree generated by the table converter consists of DictDictElements, each being
+ The subtree generated by the table converter consists of DictElements, each being
a row. The corresponding header elements will become the dictionary keys.
- The rows can be matched using a DictDictElementConverter.
+ The rows can be matched using a DictElementConverter.
"""
@abstractmethod
def get_options(self):
@@ -827,7 +1022,8 @@ class TableConverter(Converter):
if opt_name in self.definition:
el = self.definition[opt_name]
# The option can often either be a single value or a list of values.
- # In the latter case each element of the list will be converted to the defined type.
+ # In the latter case each element of the list will be converted to the defined
+ # type.
if isinstance(el, list):
option_dict[opt_name] = [
opt_conversion(el_el) for el_el in el]
@@ -838,7 +1034,9 @@ class TableConverter(Converter):
def typecheck(self, element: StructureElement):
return isinstance(element, File)
+ @Converter.debug_matching("name")
def match(self, element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("Element must be a File.")
m = re.match(self.definition["match"], element.name)
@@ -865,13 +1063,14 @@ class XLSXTableConverter(TableConverter):
def create_children(self, generalStore: GeneralStore,
element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("Element must be a File.")
table = pd.read_excel(element.path, **self.get_options())
child_elements = list()
for index, row in table.iterrows():
child_elements.append(
- DictDictElement(str(index), row.to_dict()))
+ DictElement(str(index), row.to_dict()))
return child_elements
@@ -893,11 +1092,12 @@ class CSVTableConverter(TableConverter):
def create_children(self, generalStore: GeneralStore,
element: StructureElement):
+ # TODO: See comment on types and inheritance
if not isinstance(element, File):
raise RuntimeError("Element must be a File.")
table = pd.read_csv(element.path, **self.get_options())
child_elements = list()
for index, row in table.iterrows():
child_elements.append(
- DictDictElement(str(index), row.to_dict()))
+ DictElement(str(index), row.to_dict()))
return child_elements
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index 9abbd80cb378cac38f014f5ea065358bd3c1cac3..6cf025a024e8cc392a7175421d47fb69059302a4 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -29,34 +29,42 @@ the acuired data with CaosDB.
"""
from __future__ import annotations
+
+import argparse
import importlib
-from caosadvancedtools.cache import UpdateCache, Cache
-import uuid
-import sys
+import logging
import os
+import sys
+import uuid
+import warnings
import yaml
+
+from argparse import RawTextHelpFormatter
+from collections import defaultdict
+from copy import deepcopy
from enum import Enum
-import logging
from importlib_resources import files
-import argparse
-from argparse import RawTextHelpFormatter
+from jsonschema import validate
+from typing import Any, Optional, Type, Union
+
import caosdb as db
+
+from caosadvancedtools.cache import UpdateCache, Cache
from caosadvancedtools.crawler import Crawler as OldCrawler
+from caosdb.apiutils import (compare_entities, EntityMergeConflictError,
+ merge_entities)
from caosdb.common.datatype import is_reference
-from .stores import GeneralStore, RecordStore
-from .identified_cache import IdentifiedCache
-from .structure_elements import StructureElement, Directory
-from .converters import Converter, DirectoryConverter
+
+from .converters import Converter, DirectoryConverter, ConverterValidationError
+from .identifiable import Identifiable
from .identifiable_adapters import (IdentifiableAdapter,
LocalStorageIdentifiableAdapter,
CaosDBIdentifiableAdapter)
-from collections import defaultdict
-from typing import Any, Optional, Type, Union
-from caosdb.apiutils import compare_entities, merge_entities
-from copy import deepcopy
-from jsonschema import validate
-
+from .identified_cache import IdentifiedCache
from .macros import defmacro_constructor, macro_constructor
+from .stores import GeneralStore, RecordStore
+from .structure_elements import StructureElement, Directory, NoneElement
+from .version import check_cfood_version
logger = logging.getLogger(__name__)
@@ -206,9 +214,9 @@ class Crawler(object):
if generalStore is None:
self.generalStore = GeneralStore()
- self.identifiableAdapter = identifiableAdapter
- if identifiableAdapter is None:
- self.identifiableAdapter = LocalStorageIdentifiableAdapter()
+ self.identifiableAdapter: IdentifiableAdapter = LocalStorageIdentifiableAdapter()
+ if identifiableAdapter is not None:
+ self.identifiableAdapter = identifiableAdapter
# If a directory is crawled this may hold the path to that directory
self.crawled_directory: Optional[str] = None
self.debug = debug
@@ -248,12 +256,17 @@ class Crawler(object):
if len(crawler_definitions) == 1:
# Simple case, just one document:
crawler_definition = crawler_definitions[0]
+ metadata = {}
elif len(crawler_definitions) == 2:
+ metadata = crawler_definitions[0]["metadata"] if "metadata" in crawler_definitions[0] else {
+ }
crawler_definition = crawler_definitions[1]
else:
raise RuntimeError(
"Crawler definition must not contain more than two documents.")
+ check_cfood_version(metadata)
+
# TODO: at this point this function can already load the cfood schema extensions
# from the crawler definition and add them to the yaml schema that will be
# tested in the next lines of code:
@@ -268,8 +281,8 @@ class Crawler(object):
schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
key)
if len(crawler_definitions) == 2:
- if "Converters" in crawler_definitions[0]["metadata"]:
- for key in crawler_definitions[0]["metadata"]["Converters"]:
+ if "Converters" in metadata:
+ for key in metadata["Converters"]:
schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
key)
@@ -293,6 +306,8 @@ class Crawler(object):
definition[key] = os.path.join(
os.path.dirname(definition_path), value)
if not os.path.isfile(definition[key]):
+ # TODO(henrik) capture this in `crawler_main` similar to
+ # `ConverterValidationError`.
raise FileNotFoundError(
f"Couldn't find validation file {definition[key]}")
elif isinstance(value, dict):
@@ -327,46 +342,70 @@ class Crawler(object):
"converter": "MarkdownFileConverter",
"package": "caoscrawler.converters"},
"File": {
- "converter": "FileConverter",
+ "converter": "SimpleFileConverter",
"package": "caoscrawler.converters"},
"JSONFile": {
"converter": "JSONFileConverter",
"package": "caoscrawler.converters"},
+ "YAMLFile": {
+ "converter": "YAMLFileConverter",
+ "package": "caoscrawler.converters"},
"CSVTableConverter": {
"converter": "CSVTableConverter",
"package": "caoscrawler.converters"},
"XLSXTableConverter": {
"converter": "XLSXTableConverter",
"package": "caoscrawler.converters"},
- "Dict": {
- "converter": "DictConverter",
- "package": "caoscrawler.converters"},
"DictBooleanElement": {
- "converter": "DictBooleanElementConverter",
+ "converter": "BooleanElementConverter",
+ "package": "caoscrawler.converters"},
+ "BooleanElement": {
+ "converter": "BooleanElementConverter",
"package": "caoscrawler.converters"},
"DictFloatElement": {
- "converter": "DictFloatElementConverter",
+ "converter": "FloatElementConverter",
+ "package": "caoscrawler.converters"},
+ "FloatElement": {
+ "converter": "FloatElementConverter",
"package": "caoscrawler.converters"},
"DictTextElement": {
- "converter": "DictTextElementConverter",
+ "converter": "TextElementConverter",
+ "package": "caoscrawler.converters"},
+ "TextElement": {
+ "converter": "TextElementConverter",
"package": "caoscrawler.converters"},
"DictIntegerElement": {
- "converter": "DictIntegerElementConverter",
+ "converter": "IntegerElementConverter",
+ "package": "caoscrawler.converters"},
+ "IntegerElement": {
+ "converter": "IntegerElementConverter",
"package": "caoscrawler.converters"},
"DictListElement": {
- "converter": "DictListElementConverter",
+ "converter": "ListElementConverter",
+ "package": "caoscrawler.converters"},
+ "ListElement": {
+ "converter": "ListElementConverter",
"package": "caoscrawler.converters"},
"DictDictElement": {
- "converter": "DictDictElementConverter",
+ "converter": "DictElementConverter",
+ "package": "caoscrawler.converters"},
+ "DictElement": {
+ "converter": "DictElementConverter",
+ "package": "caoscrawler.converters"},
+ "Dict": {
+ "converter": "DictElementConverter",
"package": "caoscrawler.converters"},
- "TextElement": {
- "converter": "TextElementConverter",
- "package": "caoscrawler.converters"}
}
# More converters from definition file:
if "Converters" in definition:
for key, entry in definition["Converters"].items():
+ if key in ["Dict", "DictTextElement", "DictIntegerElement", "DictBooleanElement",
+ "DictDictElement", "DictListElement", "DictFloatElement"]:
+ warnings.warn(DeprecationWarning(f"{key} is deprecated. Please use the new"
+ " variant; without 'Dict' prefix or "
+ "'DictElement' in case of 'Dict'"))
+
converter_registry[key] = {
"converter": entry["converter"],
"package": entry["package"]
@@ -461,13 +500,11 @@ class Crawler(object):
items = [items]
self.run_id = uuid.uuid1()
- local_converters = Crawler.initialize_converters(
- crawler_definition, converter_registry)
+ local_converters = Crawler.initialize_converters(crawler_definition, converter_registry)
# This recursive crawling procedure generates the update list:
self.crawled_data: list[db.Record] = []
- self._crawl(items, local_converters, self.generalStore,
- self.recordStore, [], [])
+ self._crawl(items, local_converters, self.generalStore, self.recordStore, [], [])
if self.debug:
self.debug_converters = local_converters
@@ -484,33 +521,54 @@ class Crawler(object):
return self._synchronize(self.crawled_data, commit_changes, unique_names=unique_names)
- def has_reference_value_without_id(self, record: db.Record):
+ def _has_reference_value_without_id(self, ident: Identifiable) -> bool:
"""
- Returns True if there is at least one property in `record` which:
+ Returns True if there is at least one value in the properties attribute of ``ident`` which:
+
a) is a reference property AND
- b) where the value is set to a db.Entity (instead of an ID) AND
- c) where the ID of the value (the db.Entity object in b)) is not set (to an integer)
+ b) where the value is set to a
+ :external+caosdb-pylib:py:class:`db.Entity <caosdb.common.models.Entity>`
+ (instead of an ID) AND
+ c) where the ID of the value (the
+ :external+caosdb-pylib:py:class:`db.Entity <caosdb.common.models.Entity>` object in b))
+ is not set (to an integer)
- Returns False otherwise.
+ Returns
+ -------
+ bool
+ True if there is a value without id (see above)
+
+ Raises
+ ------
+ ValueError
+ If no Identifiable is given.
"""
- for p in record.properties:
- if isinstance(p.value, list):
- for el in p.value:
+ if ident is None:
+ raise ValueError("Identifiable has to be given as argument")
+ for pvalue in list(ident.properties.values()) + ident.backrefs:
+ if isinstance(pvalue, list):
+ for el in pvalue:
if isinstance(el, db.Entity) and el.id is None:
return True
- elif isinstance(p.value, db.Entity) and p.value.id is None:
+ elif isinstance(pvalue, db.Entity) and pvalue.id is None:
return True
return False
@staticmethod
- def create_flat_list(ent_list: list[db.Entity], flat: list[db.Entity]):
+ def create_flat_list(ent_list: list[db.Entity], flat: Optional[list[db.Entity]] = None):
"""
- Recursively adds all properties contained in entities from ent_list to
- the output list flat. Each element will only be added once to the list.
+ Recursively adds entities and all their properties contained in ent_list to
+ the output list flat.
TODO: This function will be moved to pylib as it is also needed by the
high level API.
"""
+ # Note: A set would be useful here, but we do not want a random order.
+ if flat is None:
+ flat = list()
+ for el in ent_list:
+ if el not in flat:
+ flat.append(el)
for ent in ent_list:
for p in ent.properties:
# For lists append each element that is of type Entity to flat:
@@ -524,29 +582,31 @@ class Crawler(object):
if p.value not in flat:
flat.append(p.value)
Crawler.create_flat_list([p.value], flat)
+ return flat
- def has_missing_object_in_references(self, record: db.Record):
+ def _has_missing_object_in_references(self, ident: Identifiable, referencing_entities: list):
"""
- returns False if any property value is a db.Entity object that
- is contained in the `remote_missing_cache`. If the record has such an object in the
- reference properties, it means that it references another Entity, where we checked
+ returns False if any value in the properties attribute is a db.Entity object that
+ is contained in the `remote_missing_cache`. If ident has such an object in
+ properties, it means that it references another Entity, where we checked
whether it exists remotely and it was not found.
"""
- for p in record.properties:
- # if (is_reference(p)
+ if ident is None:
+ raise ValueError("Identifiable has to be given as argument")
+ for pvalue in list(ident.properties.values()) + ident.backrefs:
# Entity instead of ID and not cached locally
- if (isinstance(p.value, list)):
- for el in p.value:
- if (isinstance(el, db.Entity)
- and self.get_from_remote_missing_cache(el) is not None):
+ if (isinstance(pvalue, list)):
+ for el in pvalue:
+ if (isinstance(el, db.Entity) and self.get_from_remote_missing_cache(
+ self.identifiableAdapter.get_identifiable(el, referencing_entities)) is not None):
return True
- if (isinstance(p.value, db.Entity)
- and self.get_from_remote_missing_cache(p.value) is not None):
+ if (isinstance(pvalue, db.Entity) and self.get_from_remote_missing_cache(
+ self.identifiableAdapter.get_identifiable(pvalue, referencing_entities)) is not None):
# might be checked when reference is resolved
return True
return False
- def replace_references_with_cached(self, record: db.Record):
+ def replace_references_with_cached(self, record: db.Record, referencing_entities: list):
"""
Replace all references with the versions stored in the cache.
@@ -558,7 +618,7 @@ class Crawler(object):
for el in p.value:
if (isinstance(el, db.Entity) and el.id is None):
cached = self.get_from_any_cache(
- el)
+ self.identifiableAdapter.get_identifiable(el, referencing_entities))
if cached is None:
raise RuntimeError("Not in cache.")
if not check_identical(cached, el, True):
@@ -572,7 +632,8 @@ class Crawler(object):
lst.append(el)
p.value = lst
if (isinstance(p.value, db.Entity) and p.value.id is None):
- cached = self.get_from_any_cache(p.value)
+ cached = self.get_from_any_cache(
+ self.identifiableAdapter.get_identifiable(p.value, referencing_entities))
if cached is None:
raise RuntimeError("Not in cache.")
if not check_identical(cached, p.value, True):
@@ -583,38 +644,28 @@ class Crawler(object):
raise RuntimeError("Not identical.")
p.value = cached
- def get_from_remote_missing_cache(self, record: db.Record):
+ def get_from_remote_missing_cache(self, identifiable: Identifiable):
"""
- returns the identifiable if an identifiable with the same values already exists locally
+ returns the identified record if an identifiable with the same values already exists locally
(Each identifiable that is not found on the remote server, is 'cached' locally to prevent
that the same identifiable exists twice)
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
- identifiable = self.identifiableAdapter.get_identifiable(record)
if identifiable is None:
- # TODO: check whether the same idea as below works here
- identifiable = record
- # return None
+ raise ValueError("Identifiable has to be given as argument")
if identifiable in self.remote_missing_cache:
return self.remote_missing_cache[identifiable]
else:
return None
- def get_from_any_cache(self, record: db.Record):
+ def get_from_any_cache(self, identifiable: Identifiable):
"""
returns the identifiable if an identifiable with the same values already exists locally
(Each identifiable that is not found on the remote server, is 'cached' locally to prevent
that the same identifiable exists twice)
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
- identifiable = self.identifiableAdapter.get_identifiable(record)
if identifiable is None:
- # TODO: check whether the same idea as below works here
- identifiable = record
- # return None
+ raise ValueError("Identifiable has to be given as argument")
if identifiable in self.remote_existing_cache:
return self.remote_existing_cache[identifiable]
@@ -623,56 +674,33 @@ class Crawler(object):
else:
return None
- def add_to_remote_missing_cache(self, record: db.Record):
+ def add_to_remote_missing_cache(self, record: db.Record, identifiable: Identifiable):
"""
- adds the given identifiable to the local cache
-
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
+ stores the given Record in the remote_missing_cache.
- Return False if there is no identifiable for this record and True otherwise.
+ If identifiable is None, the Record is NOT stored.
"""
- self.add_to_cache(record=record, cache=self.remote_missing_cache)
+ self.add_to_cache(record=record, cache=self.remote_missing_cache,
+ identifiable=identifiable)
- def add_to_remote_existing_cache(self, record: db.Record):
+ def add_to_remote_existing_cache(self, record: db.Record, identifiable: Identifiable):
"""
- adds the given identifiable to the local cache
+ stores the given Record in the remote_existing_cache.
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
-
- Return False if there is no identifiable for this record and True otherwise.
+ If identifiable is None, the Record is NOT stored.
"""
- self.add_to_cache(record=record, cache=self.remote_existing_cache)
+ self.add_to_cache(record=record, cache=self.remote_existing_cache,
+ identifiable=identifiable)
- def add_to_cache(self, record: db.Record, cache):
+ def add_to_cache(self, record: db.Record, cache: IdentifiedCache,
+ identifiable: Identifiable) -> None:
"""
- adds the given identifiable to the local cache
+ stores the given Record in the given cache.
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
-
- Return False if there is no identifiable for this record and True otherwise.
+ If identifiable is None, the Record is NOT stored.
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
- identifiable = self.identifiableAdapter.get_identifiable(record)
- if identifiable is None:
- # TODO: this error report is bad
- # we need appropriate handling for records without an identifiable
- # or at least a simple fallback definition if tehre is no identifiable.
-
- # print(record)
- # raise RuntimeError("No identifiable for record.")
-
- # TODO: check whether that holds:
- # if there is no identifiable, for the cache that is the same
- # as if the complete entity is the identifiable:
- identifiable = record
- cache.add(identifiable=identifiable, record=record)
+ if identifiable is not None:
+ cache.add(identifiable=identifiable, record=record)
@staticmethod
def bend_references_to_new_object(old, new, entities):
@@ -690,14 +718,41 @@ class Crawler(object):
if p.value is old:
p.value = new
+ @staticmethod
+ def create_reference_mapping(flat: list[db.Entity]):
+ """
+ Create a dictionary of dictionaries of the form:
+ dict[int, dict[str, list[db.Entity]]]
+
+ - The integer index is the Python id of the value object.
+ - The string is the name of the first parent of the referencing object.
+
+ Each value objects is taken from the values of all properties from the list flat.
+
+ So the returned mapping maps ids of entities to the objects which are referring
+ to them.
+ """
+ # TODO we need to treat children of RecordTypes somehow.
+ references: dict[int, dict[str, list[db.Entity]]] = {}
+ for ent in flat:
+ for p in ent.properties:
+ val = p.value
+ if not isinstance(val, list):
+ val = [val]
+ for v in val:
+ if isinstance(v, db.Entity):
+ if id(v) not in references:
+ references[id(v)] = {}
+ if ent.parents[0].name not in references[id(v)]:
+ references[id(v)][ent.parents[0].name] = []
+ references[id(v)][ent.parents[0].name].append(ent)
+
+ return references
+
def split_into_inserts_and_updates(self, ent_list: list[db.Entity]):
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
to_be_inserted: list[db.Entity] = []
to_be_updated: list[db.Entity] = []
- flat = list(ent_list)
- # assure all entities are direct members TODO Can this be removed at some point?Check only?
- Crawler.create_flat_list(ent_list, flat)
+ flat = Crawler.create_flat_list(ent_list)
# TODO: can the following be removed at some point
for ent in flat:
@@ -708,61 +763,92 @@ class Crawler(object):
# flat contains Entities which could not yet be checked against the remote server
while resolved_references and len(flat) > 0:
resolved_references = False
-
+ referencing_entities = self.create_reference_mapping(
+ flat + to_be_updated + to_be_inserted)
+
+ # For each element we try to find out whether we can find it in the server or whether
+ # it does not yet exist. Since a Record may reference other unkown Records it might not
+ # be possible to answer this right away.
+ # The following checks are done on each Record:
+ # 1. Can it be identified via an ID?
+ # 2. Can it be identified via a path?
+ # 3. Is it in the cache of already checked Records?
+ # 4. Can it be checked on the remote server?
+ # 5. Does it have to be new since a needed reference is missing?
for i in reversed(range(len(flat))):
record = flat[i]
+ identifiable = self.identifiableAdapter.get_identifiable(
+ record,
+ referencing_entities=referencing_entities)
# TODO remove if the exception is never raised
- if (record.id is not None or record in to_be_inserted):
+ if record in to_be_inserted:
raise RuntimeError("This should not be reached since treated elements"
"are removed from the list")
- # Check whether this record is a duplicate that can be removed
- elif self.get_from_any_cache(record) is not None:
+ # 1. Can it be identified via an ID?
+ elif record.id is not None:
+ to_be_updated.append(record)
+ self.add_to_remote_existing_cache(record, identifiable)
+ del flat[i]
+ # 2. Can it be identified via a path?
+ elif record.path is not None:
+ existing = self._get_entity_by_path(record.path)
+ if existing is None:
+ to_be_inserted.append(record)
+ self.add_to_remote_missing_cache(record, identifiable)
+ del flat[i]
+ else:
+ record.id = existing.id
+ # TODO check the following copying of _size and _checksum
+ # Copy over checksum and size too if it is a file
+ record._size = existing._size
+ record._checksum = existing._checksum
+ to_be_updated.append(record)
+ self.add_to_remote_existing_cache(record, identifiable)
+ del flat[i]
+ # 3. Is it in the cache of already checked Records?
+ elif self.get_from_any_cache(identifiable) is not None:
# We merge the two in order to prevent loss of information
- newrecord = self.get_from_any_cache(record)
- merge_entities(newrecord, record)
+ newrecord = self.get_from_any_cache(identifiable)
+ try:
+ merge_entities(newrecord, record)
+ except EntityMergeConflictError:
+ continue
Crawler.bend_references_to_new_object(
old=record, new=newrecord, entities=flat + to_be_updated + to_be_inserted)
del flat[i]
resolved_references = True
- # can we check whether the record(identifiable) exists on the remote server?
- elif not self.has_reference_value_without_id(
- self.identifiableAdapter.get_identifiable(record)):
- # TODO: remove deepcopy?
+ # 4. Can it be checked on the remote server?
+ elif not self._has_reference_value_without_id(identifiable):
identified_record = (
- self.identifiableAdapter.retrieve_identified_record_for_record(
- deepcopy(record)))
+ self.identifiableAdapter.retrieve_identified_record_for_identifiable(
+ identifiable))
if identified_record is None:
# identifiable does not exist remotely -> record needs to be inserted
to_be_inserted.append(record)
- self.add_to_remote_missing_cache(record)
+ self.add_to_remote_missing_cache(record, identifiable)
del flat[i]
else:
# side effect
record.id = identified_record.id
- # Copy over checksum and size too if it is a file
- if isinstance(record, db.File):
- record._size = identified_record._size
- record._checksum = identified_record._checksum
-
to_be_updated.append(record)
- self.add_to_remote_existing_cache(record)
+ self.add_to_remote_existing_cache(record, identifiable)
del flat[i]
resolved_references = True
- # is it impossible to check this record because an identifiable references a
- # missing record?
- elif self.has_missing_object_in_references(
- self.identifiableAdapter.get_identifiable(record)):
+ # 5. Does it have to be new since a needed reference is missing?
+ # (Is it impossible to check this record because an identifiable references a
+ # missing record?)
+ elif self._has_missing_object_in_references(identifiable, referencing_entities):
to_be_inserted.append(record)
- self.add_to_remote_missing_cache(record)
+ self.add_to_remote_missing_cache(record, identifiable)
del flat[i]
resolved_references = True
for record in flat:
- self.replace_references_with_cached(record)
+ self.replace_references_with_cached(record, referencing_entities)
if len(flat) > 0:
raise RuntimeError(
@@ -893,6 +979,13 @@ class Crawler(object):
def _get_entity_by_name(name):
return db.Entity(name=name).retrieve()
+ @staticmethod
+ def _get_entity_by_path(path):
+ try:
+ return db.execute_query(f"FIND FILE WHICH IS STORED AT '{path}'", unique=True)
+ except db.exceptions.EmptyUniqueQueryError:
+ return None
+
@staticmethod
def _get_entity_by_id(id):
return db.Entity(id=id).retrieve()
@@ -955,10 +1048,8 @@ class Crawler(object):
Return the final to_be_inserted and to_be_updated as tuple.
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
-
to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(crawled_data)
+ referencing_entities = self.create_reference_mapping(to_be_updated + to_be_inserted)
# TODO: refactoring of typo
for el in to_be_updated:
@@ -966,8 +1057,8 @@ class Crawler(object):
self.replace_entities_with_ids(el)
identified_records = [
- self.identifiableAdapter.retrieve_identified_record_for_record(
- record)
+ self.identifiableAdapter.retrieve_identified_record_for_record(record,
+ referencing_entities)
for record in to_be_updated]
# Merge with existing data to prevent unwanted overwrites
to_be_updated = self._merge_properties_from_remote(to_be_updated,
@@ -1102,8 +1193,8 @@ ____________________\n""".format(i + 1, len(pending_changes)) + str(el[3]))
keys_modified = converter.create_records(
generalStore_copy, recordStore_copy, element)
- children = converter.create_children(
- generalStore_copy, element)
+ children = converter.create_children(generalStore_copy, element)
+
if self.debug:
# add provenance information for each varaible
self.debug_tree[str(element)] = (
@@ -1189,7 +1280,11 @@ def crawler_main(crawled_directory_path: str,
0 if successful
"""
crawler = Crawler(debug=debug, securityMode=securityMode)
- crawler.crawl_directory(crawled_directory_path, cfood_file_name)
+ try:
+ crawler.crawl_directory(crawled_directory_path, cfood_file_name)
+ except ConverterValidationError as err:
+ print(err)
+ return 1
if provenance_file is not None:
crawler.save_debug_data(provenance_file)
diff --git a/src/caoscrawler/identifiable.py b/src/caoscrawler/identifiable.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ff7172576be08e068ba412f319b059fb349bbeb
--- /dev/null
+++ b/src/caoscrawler/identifiable.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+import caosdb as db
+from datetime import datetime
+import json
+from hashlib import sha256
+from typing import Union
+
+
+class Identifiable():
+ """
+ The fingerprint of a Record in CaosDB.
+
+ This class contains the information that is used by the CaosDB Crawler to identify Records.
+ On one hand, this can be the ID or a Record or the path of a File.
+ On the other hand, in order to check whether a Record exits in the CaosDB Server, a query can
+ be created using the information contained in the Identifiable.
+
+ Parameters
+ ----------
+ record_type: str, this RecordType has to be a parent of the identified object
+ name: str, the name of the identified object
+ properties: dict, keys are names of Properties; values are Property values
+ Note, that lists are not checked for equality but are interpreted as multiple
+ conditions for a single Property.
+ path: str, In case of files: The path where the file is stored.
+ backrefs: list, TODO future
+ """
+
+ def __init__(self, record_id: int = None, path: str = None, record_type: str = None,
+ name: str = None, properties: dict = None,
+ backrefs: list[Union[int, str]] = None):
+ if (record_id is None and path is None and name is None
+ and (backrefs is None or len(backrefs) == 0)
+ and (properties is None or len(properties) == 0)):
+ raise ValueError("There is no identifying information. You need to add a path or "
+ "properties or other identifying attributes.")
+ if properties is not None and 'name' in [k.lower() for k in properties.keys()]:
+ raise ValueError("Please use the separete 'name' keyword instead of the properties "
+ "dict for name")
+ self.record_id = record_id
+ self.path = path
+ self.record_type = record_type
+ self.name = name
+ self.properties: dict = {}
+ if properties is not None:
+ self.properties = properties
+ self.backrefs: list[Union[int, db.Entity]] = []
+ if backrefs is not None:
+ self.backrefs = backrefs
+
+ def get_representation(self) -> str:
+ return sha256(Identifiable._create_hashable_string(self).encode('utf-8')).hexdigest()
+
+ @staticmethod
+ def _value_representation(value) -> str:
+ """returns the string representation of property values to be used in the hash function
+
+ The string is the path of a File Entity, the CaosDB ID or Python ID of other Entities
+ (Python Id only if there is no CaosDB ID) and the string representation of bool, float, int
+ and str.
+ """
+
+ if value is None:
+ return "None"
+ elif isinstance(value, db.File):
+ return str(value.path)
+ elif isinstance(value, db.Entity):
+ if value.id is not None:
+ return str(value.id)
+ else:
+ return "PyID=" + str(id(value))
+ elif isinstance(value, list):
+ return "[" + ", ".join([Identifiable._value_representation(el) for el in value]) + "]"
+ elif (isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
+ or isinstance(value, datetime)):
+ return str(value)
+ else:
+ raise ValueError(f"Unknown datatype of the value: {value}")
+
+ @staticmethod
+ def _create_hashable_string(identifiable: Identifiable) -> str:
+ """
+ creates a string from the attributes of an identifiable that can be hashed
+ String has the form "P<parent>N<name>R<reference-ids>a:5b:10"
+ """
+ rec_string = "P<{}>N<{}>R<{}>".format(
+ identifiable.record_type,
+ identifiable.name,
+ [Identifiable._value_representation(el) for el in identifiable.backrefs])
+ # TODO this structure neglects Properties if multiple exist for the same name
+ for pname in sorted(identifiable.properties.keys()):
+ rec_string += ("{}:".format(pname) +
+ Identifiable._value_representation(identifiable.properties[pname]))
+ return rec_string
+
+ def __eq__(self, other) -> bool:
+ """
+ Identifiables are equal if they belong to the same Record. Since ID and path are on their
+ own enough to identify the Record it is sufficient if those attributes are equal.
+ 1. both IDs are set (not None) -> equal if IDs are equal
+ 2. both paths are set (not None) -> equal if paths are equal
+ 3. equal if attribute representations are equal
+ """
+ if not isinstance(other, Identifiable):
+ raise ValueError("Identifiable can only be compared to other Identifiable objects.")
+ elif self.record_id is not None and other.record_id is not None:
+ return self.record_id == other.record_id
+ elif self.path is not None and other.path is not None:
+ return self.path == other.path
+ elif self.get_representation() == other.get_representation():
+ return True
+ else:
+ return False
+
+ def __repr__(self):
+ pstring = json.dumps(self.properties)
+ return (f"{self.__class__.__name__} for RT {self.record_type}: id={self.record_id}; "
+ f"name={self.name}\n\tpath={self.path}\n"
+ f"\tproperties:\n{pstring}\n"
+ f"\tbackrefs:\n{self.backrefs}")
diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py
index 73ce38fb593f321f9f80cc6c06034c7df0f14c76..40c801547a85afaf32e1ab6a668bc47d98d60b66 100644
--- a/src/caoscrawler/identifiable_adapters.py
+++ b/src/caoscrawler/identifiable_adapters.py
@@ -23,9 +23,12 @@
# ** end header
#
+from __future__ import annotations
import yaml
from datetime import datetime
+from typing import Any
+from .identifiable import Identifiable
import caosdb as db
import logging
from abc import abstractmethod, ABCMeta
@@ -33,14 +36,14 @@ from .utils import has_parent
logger = logging.getLogger(__name__)
-def convert_value(value):
+def convert_value(value: Any):
""" Returns a string representation of the value that is suitable
to be used in the query
looking for the identified record.
Parameters
----------
- value : The property of which the value shall be returned.
+ value : Any type, the value that shall be returned and potentially converted.
Returns
-------
@@ -52,11 +55,13 @@ def convert_value(value):
return str(value.id)
elif isinstance(value, datetime):
return value.isoformat()
- elif type(value) == str:
+ elif isinstance(value, bool):
+ return str(value).upper()
+ elif isinstance(value, str):
# replace single quotes, otherwise they may break the queries
return value.replace("\'", "\\'")
else:
- return f"{value}"
+ return str(value)
class IdentifiableAdapter(metaclass=ABCMeta):
@@ -86,7 +91,7 @@ class IdentifiableAdapter(metaclass=ABCMeta):
"""
@staticmethod
- def create_query_for_identifiable(ident: db.Record):
+ def create_query_for_identifiable(ident: Identifiable):
"""
This function is taken from the old crawler:
caosdb-advanced-user-tools/src/caosadvancedtools/crawler.py
@@ -95,34 +100,34 @@ class IdentifiableAdapter(metaclass=ABCMeta):
whether the required record already exists.
"""
- if len(ident.parents) != 1:
- raise RuntimeError(
- "Multiple parents for identifiables not supported.")
+ query_string = "FIND RECORD "
+ if ident.record_type is not None:
+ query_string += ident.record_type
+ for ref in ident.backrefs:
+ eid = ref
+ if isinstance(ref, db.Entity):
+ eid = ref.id
+ query_string += (" WHICH IS REFERENCED BY " + str(eid) + " AND")
- query_string = "FIND Record " + ident.get_parents()[0].name
query_string += " WITH "
- if ident.name is None and len(ident.get_properties()) == 0:
- raise ValueError(
- "The identifiable must have features to identify it.")
-
if ident.name is not None:
query_string += "name='{}'".format(ident.name)
- if len(ident.get_properties()) > 0:
+ if len(ident.properties) > 0:
query_string += " AND "
query_string += IdentifiableAdapter.create_property_query(ident)
return query_string
@staticmethod
- def create_property_query(entity: db.Entity):
+ def create_property_query(entity: Identifiable):
query_string = ""
- for p in entity.get_properties():
- if p.value is None:
- query_string += "'" + p.name + "' IS NULL AND "
- elif isinstance(p.value, list):
- for v in p.value:
- query_string += ("'" + p.name + "'='" +
+ for pname, pvalue in entity.properties.items():
+ if pvalue is None:
+ query_string += "'" + pname + "' IS NULL AND "
+ elif isinstance(pvalue, list):
+ for v in pvalue:
+ query_string += ("'" + pname + "'='" +
convert_value(v) + "' AND ")
# TODO: (for review)
@@ -136,8 +141,8 @@ class IdentifiableAdapter(metaclass=ABCMeta):
# IdentifiableAdapter.create_property_query(p.value) +
# ") AND ")
else:
- query_string += ("'" + p.name + "'='" +
- convert_value(p.value) + "' AND ")
+ query_string += ("'" + pname + "'='" +
+ convert_value(pvalue) + "' AND ")
# remove the last AND
return query_string[:-4]
@@ -160,79 +165,90 @@ class IdentifiableAdapter(metaclass=ABCMeta):
"""
pass
- def get_identifiable_for_file(self, record: db.File):
- """
- Retrieve an identifiable for a file.
-
- Currently an identifiable for a file ist just a File object
- with a specific path. In the future, this could be extended
- to allow for names, parents and custom properties.
- """
- identifiable = db.File()
- identifiable.path = record.path
- return identifiable
-
- def get_identifiable(self, record: db.Record):
+ def get_identifiable(self, record: db.Record, referencing_entities=None):
"""
retrieve the registred identifiable and fill the property values to create an
identifiable
- """
- if record.role == "File":
- return self.get_identifiable_for_file(record)
+ Args:
+ record: the record for which the Identifiable shall be created.
+ referencing_entities: a dictionary (Type: dict[int, dict[str, list[db.Entity]]]), that
+ allows to look up entities with a certain RecordType, that reference ``record``
+
+ Returns:
+ Identifiable, the identifiable for record.
+ """
registered_identifiable = self.get_registered_identifiable(record)
- if registered_identifiable is None:
- return None
+ if referencing_entities is None:
+ referencing_entities = {}
- identifiable = db.Record(name=record.name)
- if len(registered_identifiable.parents) != 1:
- raise RuntimeError("Multiple parents for identifiables"
- "not supported.")
- identifiable.add_parent(registered_identifiable.parents[0])
property_name_list_A = []
property_name_list_B = []
-
- # fill the values:
- for prop in registered_identifiable.properties:
- if prop.name == "name":
- # The name can be an identifiable, but it isn't a property
- continue
- # problem: what happens with multi properties?
- # case A: in the registered identifiable
- # case B: in the identifiable
-
- record_prop = record.get_property(prop.name)
- if record_prop is None:
- # TODO: how to handle missing values in identifiables
- # raise an exception?
- raise NotImplementedError(
- f"The following record is missing an identifying property:"
- f"RECORD\n{record}\nIdentifying PROPERTY\n{prop.name}"
- )
- newval = record_prop.value
- record_prop_new = db.Property(name=record_prop.name,
- id=record_prop.id,
- description=record_prop.description,
- datatype=record_prop.datatype,
- value=newval,
- unit=record_prop.unit)
- identifiable.add_property(record_prop_new)
- property_name_list_A.append(prop.name)
-
- # check for multi properties in the record:
- for prop in property_name_list_A:
- property_name_list_B.append(prop)
- if (len(set(property_name_list_B)) != len(property_name_list_B) or len(
- set(property_name_list_A)) != len(property_name_list_A)):
- raise RuntimeError(
- "Multi properties used in identifiables can cause unpredictable results.")
-
- return identifiable
+ identifiable_props = {}
+ identifiable_backrefs = []
+
+ if registered_identifiable is not None:
+ # fill the values:
+ for prop in registered_identifiable.properties:
+ if prop.name == "name":
+ # The name can be an identifiable, but it isn't a property
+ continue
+ # problem: what happens with multi properties?
+ # case A: in the registered identifiable
+ # case B: in the identifiable
+
+ # TODO: similar to the Identifiable class, Registred Identifiable should be a
+ # separate class too
+ if prop.name.lower() == "is_referenced_by":
+ for rtname in prop.value:
+ if (id(record) in referencing_entities
+ and rtname in referencing_entities[id(record)]):
+ identifiable_backrefs.extend(referencing_entities[id(record)][rtname])
+ else:
+ # TODO: is this the appropriate error?
+ raise NotImplementedError(
+ f"The following record is missing an identifying property:"
+ f"RECORD\n{record}\nIdentifying PROPERTY\n{prop.name}"
+ )
+ continue
+
+ record_prop = record.get_property(prop.name)
+ if record_prop is None:
+ # TODO: how to handle missing values in identifiables
+ # raise an exception?
+ # TODO: is this the appropriate error?
+ raise NotImplementedError(
+ f"The following record is missing an identifying property:"
+ f"RECORD\n{record}\nIdentifying PROPERTY\n{prop.name}"
+ )
+ identifiable_props[record_prop.name] = record_prop.value
+ property_name_list_A.append(prop.name)
+
+ # check for multi properties in the record:
+ for prop in property_name_list_A:
+ property_name_list_B.append(prop)
+ if (len(set(property_name_list_B)) != len(property_name_list_B) or len(
+ set(property_name_list_A)) != len(property_name_list_A)):
+ raise RuntimeError(
+ "Multi properties used in identifiables could cause unpredictable results and "
+ "are not allowed. You might want to consider a Property with a list as value.")
+
+ # use the RecordType of the registred Identifiable if it exists
+ # We do not use parents of Record because it might have multiple
+ return Identifiable(
+ record_id=record.id,
+ record_type=(registered_identifiable.parents[0].name
+ if registered_identifiable else None),
+ name=record.name,
+ properties=identifiable_props,
+ path=record.path,
+ backrefs=identifiable_backrefs
+ )
@abstractmethod
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
"""
Retrieve identifiable record for a given identifiable.
@@ -245,7 +261,7 @@ class IdentifiableAdapter(metaclass=ABCMeta):
# TODO: remove side effect
# TODO: use ID if record has one?
- def retrieve_identified_record_for_record(self, record: db.Record):
+ def retrieve_identified_record_for_record(self, record: db.Record, referencing_entities=None):
"""
This function combines all functionality of the IdentifierAdapter by
returning the identifiable after having checked for an appropriate
@@ -254,12 +270,9 @@ class IdentifiableAdapter(metaclass=ABCMeta):
In case there was no appropriate registered identifiable or no identifiable could
be found return value is None.
"""
- identifiable = self.get_identifiable(record)
-
- if identifiable is None:
- return None
+ identifiable = self.get_identifiable(record, referencing_entities=referencing_entities)
- if identifiable.role == "File":
+ if identifiable.path is not None:
return self.get_file(identifiable)
return self.retrieve_identified_record_for_identifiable(identifiable)
@@ -280,7 +293,7 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
def get_records(self):
return self._records
- def get_file(self, identifiable: db.File):
+ def get_file(self, identifiable: Identifiable):
"""
Just look in records for a file with the same path.
"""
@@ -338,7 +351,7 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
return None
return identifiable_candidates[0]
- def check_record(self, record: db.Record, identifiable: db.Record):
+ def check_record(self, record: db.Record, identifiable: Identifiable):
"""
Check for a record from the local storage (named "record") if it is
the identified record for an identifiable which was created by
@@ -348,13 +361,11 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
record is the record from the local database to check against.
identifiable is the record that was created during the crawler run.
"""
- if len(identifiable.parents) != 1:
- raise RuntimeError(
- "Multiple parents for identifiables not supported.")
- if not has_parent(record, identifiable.parents[0].name):
+ if (identifiable.record_type is not None
+ and not has_parent(record, identifiable.record_type)):
return False
- for prop in identifiable.properties:
- prop_record = record.get_property(prop.name)
+ for propname, propvalue in identifiable.properties.items():
+ prop_record = record.get_property(propname)
if prop_record is None:
return False
@@ -362,18 +373,18 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
# there are two different cases:
# a) prop_record.value has a registered identifiable:
# in this case, fetch the identifiable and set the value accordingly
- if isinstance(prop.value, db.Entity): # lists are not checked here
+ if isinstance(propvalue, db.Entity): # lists are not checked here
otherid = prop_record.value
if isinstance(prop_record.value, db.Entity):
otherid = prop_record.value.id
- if prop.value.id != otherid:
+ if propvalue.id != otherid:
return False
- elif prop.value != prop_record.value:
+ elif propvalue != prop_record.value:
return False
return True
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
candidates = []
for record in self._records:
if self.check_record(record, identifiable):
@@ -420,13 +431,20 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
for key, value in identifiable_data.items():
rt = db.RecordType().add_parent(key)
for prop_name in value:
- rt.add_property(name=prop_name)
+ if isinstance(prop_name, str):
+ rt.add_property(name=prop_name)
+ elif isinstance(prop_name, dict):
+ for k, v in prop_name.items():
+ rt.add_property(name=k, value=v)
+ else:
+ NotImplementedError("YAML is not structured correctly")
+
self.register_identifiable(key, rt)
def register_identifiable(self, name: str, definition: db.RecordType):
self._registered_identifiables[name] = definition
- def get_file(self, identifiable: db.File):
+ def get_file(self, identifiable: Identifiable):
if identifiable.path is None:
raise RuntimeError("Path must not be None for File retrieval.")
candidates = db.execute_query("FIND File which is stored at {}".format(
@@ -444,6 +462,9 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
It is assumed, that there is exactly one identifiable for each RecordType. Only the first
parent of the given Record is considered; others are ignored
"""
+ if len(record.parents) == 0:
+ return None
+ # TODO We need to treat the case where multiple parents exist properly.
rt_name = record.parents[0].name
for name, definition in self._registered_identifiables.items():
if definition.parents[0].name.lower() == rt_name.lower():
@@ -458,12 +479,14 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
return record
return record.id
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
query_string = self.create_query_for_identifiable(identifiable)
candidates = db.execute_query(query_string)
if len(candidates) > 1:
raise RuntimeError(
- f"Identifiable was not defined unambigiously.\n{query_string}\nReturned the following {candidates}.")
+ f"Identifiable was not defined unambigiously.\n{query_string}\nReturned the "
+ f"following {candidates}."
+ f"Identifiable:\n{identifiable.record_type}{identifiable.properties}")
if len(candidates) == 0:
return None
return candidates[0]
diff --git a/src/caoscrawler/identified_cache.py b/src/caoscrawler/identified_cache.py
index b5db8ba04fc813ea8aca0131c6265adab666b2e7..aa2d82f8e66c738e737c62f3cc68eaf60127e28b 100644
--- a/src/caoscrawler/identified_cache.py
+++ b/src/caoscrawler/identified_cache.py
@@ -25,87 +25,41 @@
"""
-This module is a cache for Records where we checked the existence in a remote server using
-identifiables. If the Record was found, this means that we identified the corresponding Record
-in the remote server and the ID of the local object can be set.
-To prevent querying the server again and again for the same objects, this cache allows storing
-Records that were found on a remote server and those that were not (typically in separate caches).
-The look up in the cache is done using a hash of a string representation.
-
-TODO: We need a general review:
-- How are entities identified with each other?
-- What happens if the identification fails?
-
-Checkout how this was done in the old crawler.
+see class docstring
"""
+from .identifiable import Identifiable
import caosdb as db
-from hashlib import sha256
-from datetime import datetime
-
-
-def _value_representation(value):
- """returns the string representation of property values to be used in the hash function """
-
- # TODO: (for review)
- # This expansion of the hash function was introduced recently
- # to allow the special case of Files as values of properties.
- # We need to review the completeness of all the cases here, as the cache
- # is crucial for correct identification of insertion and updates.
- if value is None:
- return "None"
- elif isinstance(value, db.File):
- return str(value.path)
- elif isinstance(value, db.Entity):
- if value.id is not None:
- return str(value.id)
- else:
- return "PyID="+str(id(value))
- elif isinstance(value, list):
- return "["+", ".join([_value_representation(el) for el in value])+"]"
- elif (isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
- or isinstance(value, datetime)):
- return str(value)
- else:
- raise ValueError(f"Unknown datatype of the value: {value}")
-
-def _create_hashable_string(identifiable: db.Record):
+class IdentifiedCache(object):
"""
- creates a string from the attributes of an identifiable that can be hashed
+ This class is like a dictionary where the keys are Identifiables. When you check whether an
+ Identifiable exists as key this class returns True not only if that exact Python object is
+ used as a key, but if an Identifiable is used as key that is **equal** to the one being
+ considered (see __eq__ function of Identifiable). Similarly, if you do `cache[identifiable]`
+ you get the Record where the key is an Identifiable that is equal to the one in the rectangular
+ brackets.
+
+ This class is used for Records where we checked the existence in a remote server using
+ identifiables. If the Record was found, this means that we identified the corresponding Record
+ in the remote server and the ID of the local object can be set.
+ To prevent querying the server again and again for the same objects, this cache allows storing
+ Records that were found on a remote server and those that were not (typically in separate
+ caches).
"""
- if identifiable.role == "File":
- # Special treatment for files:
- return "P<>N<>{}:{}".format("path", identifiable.path)
- if len(identifiable.parents) != 1:
- # TODO: extend this
- # maybe something like this:
- # parent_names = ",".join(
- # sorted([p.name for p in identifiable.parents])
- raise RuntimeError("Cache entry can only be generated for entities with 1 parent.")
- rec_string = "P<{}>N<{}>".format(identifiable.parents[0].name, identifiable.name)
- # TODO this structure neglects Properties if multiple exist for the same name
- for pname in sorted([p.name for p in identifiable.properties]):
-
- rec_string += ("{}:".format(pname) +
- _value_representation(identifiable.get_property(pname).value))
- return rec_string
-
-def _create_hash(identifiable: db.Record) -> str:
- return sha256(_create_hashable_string(identifiable).encode('utf-8')).hexdigest()
-
-
-class IdentifiedCache(object):
def __init__(self):
self._cache = {}
+ self._identifiables = []
- def __contains__(self, identifiable: db.Record):
- return _create_hash(identifiable) in self._cache
+ def __contains__(self, identifiable: Identifiable):
+ return identifiable in self._identifiables
def __getitem__(self, identifiable: db.Record):
- return self._cache[_create_hash(identifiable)]
+ index = self._identifiables.index(identifiable)
+ return self._cache[id(self._identifiables[index])]
- def add(self, record: db.Record, identifiable: db.Record):
- self._cache[_create_hash(identifiable)] = record
+ def add(self, record: db.Record, identifiable: Identifiable):
+ self._cache[id(identifiable)] = record
+ self._identifiables.append(identifiable)
diff --git a/src/caoscrawler/macros/macro_yaml_object.py b/src/caoscrawler/macros/macro_yaml_object.py
index 2849986e6deb5cb2cba9e45516e6ce8e1a93dfa0..c6b5de27d7f498d9b1db6b6a90d986487340a880 100644
--- a/src/caoscrawler/macros/macro_yaml_object.py
+++ b/src/caoscrawler/macros/macro_yaml_object.py
@@ -135,6 +135,7 @@ def macro_constructor(loader, node):
raise RuntimeError("params type not supported")
else:
raise RuntimeError("params type must not be None")
+ params = substitute_dict(params, params)
definition = substitute_dict(macro.definition, params)
res.update(definition)
else:
@@ -146,6 +147,7 @@ def macro_constructor(loader, node):
params.update(params_setter)
else:
raise RuntimeError("params type not supported")
+ params = substitute_dict(params, params)
definition = substitute_dict(macro.definition, params)
res.update(definition)
else:
diff --git a/src/caoscrawler/structure_elements.py b/src/caoscrawler/structure_elements.py
index 01996b4ff3e14a9739857e6e03ceca161300b37e..952f29d012f8373062ed9dfe8a830bd18c4b0baa 100644
--- a/src/caoscrawler/structure_elements.py
+++ b/src/caoscrawler/structure_elements.py
@@ -23,7 +23,8 @@
# ** end header
#
-from typing import Dict
+from typing import Dict as tDict
+import warnings
class StructureElement(object):
@@ -31,7 +32,7 @@ class StructureElement(object):
def __init__(self, name):
# Used to store usage information for debugging:
- self.metadata: Dict[str, set[str]] = {
+ self.metadata: tDict[str, set[str]] = {
"usage": set()
}
@@ -55,6 +56,10 @@ class FileSystemStructureElement(StructureElement):
return "{}: {}, {}".format(class_name_short, self.name, self.path)
+class NoneElement(StructureElement):
+ pass
+
+
class Directory(FileSystemStructureElement):
pass
@@ -68,48 +73,78 @@ class JSONFile(File):
class DictElement(StructureElement):
- def __init__(self, name: str, value):
+ def __init__(self, name: str, value: dict):
super().__init__(name)
self.value = value
-class Dict(StructureElement):
- def __init__(self, name: str, value: dict):
+class TextElement(StructureElement):
+ def __init__(self, name: str, value: str):
super().__init__(name)
self.value = value
-class DictTextElement(DictElement):
- def __init__(self, name: str, value: str):
- super().__init__(name, value)
+class DictTextElement(TextElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use TextElement."))
+ super().__init__(*args, **kwargs)
-class DictIntegerElement(DictElement):
+class IntegerElement(StructureElement):
def __init__(self, name: str, value: int):
- super().__init__(name, value)
+ super().__init__(name)
+ self.value = value
+
+class DictIntegerElement(IntegerElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use IntegerElement."))
+ super().__init__(*args, **kwargs)
-class DictBooleanElement(DictElement):
+
+class BooleanElement(StructureElement):
def __init__(self, name: str, value: bool):
- super().__init__(name, value)
+ super().__init__(name)
+ self.value = value
-class DictDictElement(Dict, DictElement):
- def __init__(self, name: str, value: dict):
- DictElement.__init__(self, name, value)
+class DictBooleanElement(BooleanElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use BooleanElement."))
+ super().__init__(*args, **kwargs)
-class DictListElement(DictElement):
- def __init__(self, name: str, value: dict):
- super().__init__(name, value)
+class ListElement(StructureElement):
+ def __init__(self, name: str, value: list):
+ super().__init__(name)
+ self.value = value
-class DictFloatElement(DictElement):
- def __init__(self, name: str, value: float):
- super().__init__(name, value)
+class DictListElement(ListElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use ListElement."))
+ super().__init__(*args, **kwargs)
-class TextElement(StructureElement):
- def __init__(self, name: str, value: str):
+class FloatElement(StructureElement):
+ def __init__(self, name: str, value: float):
super().__init__(name)
self.value = value
+
+
+class DictFloatElement(FloatElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use FloatElement."))
+ super().__init__(*args, **kwargs)
+
+
+class Dict(DictElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use DictElement."))
+ super().__init__(*args, **kwargs)
+
+
+class DictDictElement(DictElement):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use DictElement."))
+ super().__init__(*args, **kwargs)
diff --git a/src/caoscrawler/version.py b/src/caoscrawler/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..e73905dcd25673eae88f718a7e45b7b4d0665e47
--- /dev/null
+++ b/src/caoscrawler/version.py
@@ -0,0 +1,87 @@
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+try:
+ from importlib import metadata as importlib_metadata
+except ImportError: # Python<3.8 dowesn"t support this so use
+ import importlib_metadata
+
+from packaging.version import parse as parse_version
+from warnings import warn
+
+# Read in version of locally installed caoscrawler package
+version = importlib_metadata.version("caoscrawler")
+
+
+class CfoodRequiredVersionError(RuntimeError):
+ """The installed crawler version is older than the version specified in the
+ cfood's metadata.
+
+ """
+
+
+def check_cfood_version(metadata: dict):
+
+ if not metadata or "crawler-version" not in metadata:
+
+ msg = """
+No crawler version specified in cfood definition, so there is now guarantee that
+the cfood definition matches the installed crawler version.
+
+Specifying a version is highly recommended to ensure that the definition works
+as expected with the installed version of the crawler.
+ """
+
+ warn(msg, UserWarning)
+ return
+
+ installed_version = parse_version(version)
+ cfood_version = parse_version(metadata["crawler-version"])
+
+ if cfood_version > installed_version:
+ msg = f"""
+Your cfood definition requires a newer version of the CaosDB crawler. Please
+update the crawler to the required version.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+ """
+ raise CfoodRequiredVersionError(msg)
+
+ elif cfood_version < installed_version:
+ # only warn if major or minor of installed version are newer than
+ # specified in cfood
+ if (cfood_version.major < installed_version.major) or (cfood_version.minor < installed_version.minor):
+ msg = f"""
+The cfood was written for a previous crawler version. Running the crawler in a
+newer version than specified in the cfood definition may lead to unwanted or
+unexpected behavior. Please visit the CHANGELOG
+(https://gitlab.com/caosdb/caosdb-crawler/-/blob/main/CHANGELOG.md) and check
+for any relevant changes.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+ """
+ warn(msg, UserWarning)
+ return
+
+ # At this point, the version is either equal or the installed crawler
+ # version is newer just by an increase in the patch version, so still
+ # compatible. We can safely ...
+ return
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
index b6995c9a2d950ecd1e832d5b49dac9ed88a7e455..1f6e15d408e10e38bce0d9b9fe9b6197ec69bfc3 100644
--- a/src/doc/README_SETUP.md
+++ b/src/doc/README_SETUP.md
@@ -2,9 +2,6 @@
## Installation ##
-### Requirements ###
-
-
### How to install ###
#### Linux ####
@@ -59,17 +56,12 @@ pip3 install --user .
**Note**: In the near future, this package will also be made available on PyPi.
-## Configuration ##
-
-
-
-## Try it out ##
-
-
## Run Unit Tests
## Documentation ##
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
Build documentation in `src/doc` with `make html`.
@@ -79,4 +71,10 @@ Build documentation in `src/doc` with `make html`.
- `sphinx-autoapi`
- `recommonmark`
-### Troubleshooting ###
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
+
+
diff --git a/src/doc/cfood.rst b/src/doc/cfood.rst
index 677cadc55709c6c25d16ff547b311102ee78699a..37f6a8c7d3be9298ec965c50a4ec29110988ddc6 100644
--- a/src/doc/cfood.rst
+++ b/src/doc/cfood.rst
@@ -16,6 +16,9 @@ document together with the metadata and :doc:`macro<macros>` definitions (see :r
If metadata and macro definitions are provided, there **must** be a second document preceeding the
converter tree specification, including these definitions.
+It is highly recommended to specify the version of the CaosDB crawler for which
+the cfood is written in the metadata section, see :ref:`below<example_3>`.
+
Examples
++++++++
@@ -69,6 +72,7 @@ two custom converters in the second document (**not recommended**, see the recom
metadata:
name: Datascience CFood
description: CFood for data from the local data science work group
+ crawler-version: 0.2.1
macros:
- !defmacro
name: SimulationDatasetFile
@@ -108,6 +112,7 @@ The **recommended way** of defining metadata, custom converters, macros and the
metadata:
name: Datascience CFood
description: CFood for data from the local data science work group
+ crawler-version: 0.2.1
macros:
- !defmacro
name: SimulationDatasetFile
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
index c0f21cbaa322caddabed8e045f7b6fc4253d2959..89757f21958f3d94649b33e9f9112593f703191d 100644
--- a/src/doc/concepts.rst
+++ b/src/doc/concepts.rst
@@ -11,7 +11,8 @@ of the existing data (For example could a tree of Python file objects
(StructureElements) represent a file tree that exists on some file server).
Relevant sources in:
-src/structure_elements.py
+
+- ``src/structure_elements.py``
Converters
++++++++++
@@ -22,20 +23,70 @@ the above named tree. The definition of a Converter also contains what
Converters shall be used to treat the generated child-StructureElements. The
definition is therefore a tree itself.
-See `:doc:converters<converters>` for details.
+See :std:doc:`converters<converters>` for details.
Relevant sources in:
-src/converters.py
+- ``src/converters.py``
Identifiables
+++++++++++++
-Relevant sources in:
-src/identifiable_adapters.py
+An Identifiable of a Record is like the fingerprint of a Record.
+
+The identifiable contains the information that is used by the CaosDB Crawler to identify Records.
+For example, in order to check whether a Record exits in the CaosDB Server, the CaosDB Crawler creates a query
+using the information contained in the Identifiable.
+
+Suppose a certain experiment is at most done once per day, then the identifiable could
+consist of the RecordType "SomeExperiment" (as a parent) and the Property "date" with the respective value.
+
+You can think of the properties that are used by the identifiable as a dictionary. For each property
+name there can be one value. However, this value can be a list such that the created query can look
+like "FIND RECORD ParamenterSet WITH a=5 AND a=6". This is meaningful if there is a ParamenterSet
+with two Properties with the name 'a' (multi property) or if 'a' is a list containing at least the values 5 and 6.
+
+When we use a reference Property in the identifiable, we effectively use the reference from the object to
+be identified pointing to some other object as an identifying attribute. We can also use references that point
+in the other direction, i.e. towards the object to be identified. An identifiable may denote one or more
+Entities that are referencing the object to be identified.
+
+The path of a File object can serve as a Property that identifies files and similarly the name of
+Records can be used.
+
+In the current implementation an identifiable can only use one RecordType even though the identified Records might have multiple Parents.
+
+Relevant sources in
+
+- ``src/identifiable_adapters.py``
+- ``src/identifiable.py``
+
+Registered Identifiables
+++++++++++++++++++++++++
+A Registered Identifiable is the blue print for Identifiables.
+You can think of registered identifiables as identifiables without concrete values for properties.
+RegisteredIdentifiables are
+associated with RecordTypes and define of what information an identifiable for that RecordType
+exists. There can be multiple Registered Identifiables for one RecordType.
+
+If identifiables shall contain references to the object to be identified, the Registered
+Identifiable must list the RecordTypes of the Entities that have those references.
+For example, the Registered Identifiable for the "Experiment" RecordType may contain
+the "date" Property and "Project" as the RecordType of an Entity that is referencing
+the object to be identified. Then if we have a structure of some Records at hand,
+we can check whether a Record with the parent "Project" is referencing the "Experiment"
+Record. If that is the case, this reference is part of the identifiable for the "Experiment"
+Record. Note, that if there are multiple Records with the appropriate parent (e.g.
+multiple "Project" Records in the above example) it will be required that all of them
+reference the object to be identified.
+
+
+Identified Records
+++++++++++++++++++
+TODO
The Crawler
+++++++++++
@@ -45,7 +96,8 @@ The crawler can be considered the main program doing the synchronization in basi
#. Compare the current state of the CaosDB instance with the set of CaosDB Entities created in step 1, taking into account the :ref:`registered identifiables<Identifiables>`. Insert or update entites accordingly.
Relevant sources in:
-src/crawl.py
+
+- ``src/crawl.py``
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 091cdf74a2dc11540e60b821a30e91214931c8f9..b8d055abe682efcb17f960cdaabca3de4d25a16d 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -33,10 +33,10 @@ copyright = '2021, MPIDS'
author = 'Alexander Schlemmer'
# The short X.Y version
-version = '0.2'
+version = '0.3.0'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.2'
+release = '0.3.0'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/converters.rst b/src/doc/converters.rst
index 5ca37f27f19fe5061c0291a209390daac39d5fb8..b4ba89ced3b5858ca2f8abe7bc724d6710d9203b 100644
--- a/src/doc/converters.rst
+++ b/src/doc/converters.rst
@@ -15,8 +15,6 @@ Converters may define additional functions that create further values. For
example, a regular expresion could be used to get a date from a file name.
-
-
A converter is defined via a yml file or part of it. The definition states
what kind of StructureElement it treats (typically one).
Also, it defines how children of the current StructureElement are
@@ -64,24 +62,45 @@ Standard Converters
Directory Converter
===================
+The Directory Converter creates StructureElements for each File and Directory
+inside the current Directory. You can match a regular expression against the
+directory name using the 'match' key.
Simple File Converter
=====================
+The Simple File Converter does not create any children and is usually used if
+a file shall be used as it is and be inserted and referenced by other entities.
Markdown File Converter
=======================
+Reads a YAML header from Markdown files (if such a header exists) and creates
+children elements according to the structure of the header.
-Dict Converter
+DictElement Converter
==============
+Creates a child StructureElement for each key in the dictionary.
Typical Subtree converters
--------------------------
+The following StructureElement are typically created:
+
+- BooleanElement
+- FloatElement
+- TextElement
+- IntegerElement
+- ListElement
+- DictElement
+
+Scalar Value Converters
+=======================
+`BooleanElementConverter`, `FloatElementConverter`, `TextElementConverter`, and
+`IntegerElementConverter` behave very similarly.
These converters expect `match_name` and `match_value` in their definition
which allow to match the key and the value, respectively.
Note that there are defaults for accepting other types. For example,
-DictFloatElementConverter also accepts DictIntegerElements. The default
+FloatElementConverter also accepts IntegerElements. The default
behavior can be adjusted with the fields `accept_text`, `accept_int`,
`accept_float`, and `accept_bool`.
@@ -109,8 +128,6 @@ JSONFileConverter
-TextElementConverter
-====================
TableConverter
==============
@@ -466,3 +483,22 @@ Let's formulate that using `create_records` (again, `dir_name` is constant here)
keys_modified = create_records(values, records,
record_def)
+Debugging
+=========
+
+You can add the key `debug_match` to the definition of a Converter in order to create debugging
+output for the match step. The following snippet illustrates this:
+
+.. code-block:: yaml
+
+ DirConverter:
+ type: Directory
+ match: (?P<dir_name>.*)
+ debug_match: True
+ records:
+ Project:
+ identifier: project_name
+
+
+Whenever this Converter tries to match a StructureElement, it logs what was tried to macht against
+what and what the result was.
diff --git a/src/doc/how-to-upgrade.md b/src/doc/how-to-upgrade.md
new file mode 100644
index 0000000000000000000000000000000000000000..931fa0cd2f2d621c89c35046d6df4ba6ac9b7a1e
--- /dev/null
+++ b/src/doc/how-to-upgrade.md
@@ -0,0 +1,35 @@
+
+# How to upgrade
+
+## 0.2.x to 0.3.0
+DictElementConverter (old: DictConverter) now can use "match" keywords. If
+none are in the definition, the behavior is as before. If you had "match",
+"match_name" or "match_value" in the definition of a
+
+DictConverter (StructureElement: Dict) before,
+you probably want to remove those. They were ignored before and are now used.
+
+TextElement used the 'match' keyword before, which was applied to the
+value. This is will in future be applied to the key instead and is now
+forbidden to used. If you used the 'match'
+keyword in the definition of TextElementConverter
+(StructureElement: TextElement) before, you need to change the key from "match"
+to "match_name" in order to preserve the behavior.
+
+The JSONFileConverter was changed such that it creates StructureElements as children corresponding
+to the content. I.e. if there is an Object (dict in Python) in the file, the children will be a list
+with one DictElement. Before, only JSON files with one Object were accepted and the key value pairs
+of the dict were directly transformed to children. This means, that all previously used
+JSONFileConverters need to introduce a new level for the DictElement:
+
+```
+ json:
+ type: JSONFile
+ match: metadata.json
+ validate: schema/dataset.schema.json
+ subtree:
+ jsondict: # new
+ type: DictElement # new
+ match: .* # new
+ subtree: # new
+```
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 724bcc543dd1cf0b9af451c487b1b3aab7fa95ca..b4e30e4728068cabb92626cfac986ab858a0bbb6 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -1,5 +1,5 @@
-Crawler 2.0 Documentation
-=========================
+CaosDB-Crawler Documentation
+============================
.. toctree::
@@ -13,22 +13,23 @@ Crawler 2.0 Documentation
CFoods (Crawler Definitions)<cfood>
Macros<macros>
Tutorials<tutorials/index>
+ How to upgrade<how-to-upgrade>
API documentation<_apidoc/modules>
-This is the documentation for the crawler (previously known as crawler 2.0) for CaosDB, ``caosdb-crawler``.
+This is the documentation for CaosDB-Crawler (previously known as crawler 2.0)
+the main tool for automatic data insertion into CaosDB.
-The crawler is the main date integration tool for CaosDB.
Its task is to automatically synchronize data found on file systems or in other
sources of data with the semantic data model of CaosDB.
More specifically, data that is contained in a hierarchical structure is converted to a data
structure that is consistent with a predefined semantic data model.
-The hierarchical sturcture can be for example a file tree. However it can be
-also something different like the contents of a json file or a file tree with
-json files.
+The hierarchical structure can be for example a file tree. However it can be
+also something different like the contents of a JSON file or a file tree with
+JSON files.
This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials/index>`.
@@ -40,5 +41,3 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
-
diff --git a/src/doc/tutorials/example.rst b/src/doc/tutorials/example.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a1adee7008f3b004e6b441573798b2e57f9a4384
--- /dev/null
+++ b/src/doc/tutorials/example.rst
@@ -0,0 +1,108 @@
+Example CFood
+=============
+
+Let's walk through an example cfood that makes use of a simple directory structure. We assume
+the structure which is supposed to be crawled to have the following form:
+
+.. code-block::
+
+ ExperimentalData/
+
+ 2022_ProjectA/
+
+ 2022-02-17_TestDataset/
+ file1.dat
+ file2.dat
+ ...
+ ...
+
+ 2023_ProjectB/
+ ...
+
+ ...
+
+This file structure conforms to the one described in our article "Guidelines for a Standardized Filesystem Layout for Scientific Data" (https://doi.org/10.3390/data5020043). As a simplified example
+we want to write a crawler that creates "Project" and "Measurement" records in CaosDB and set
+some reasonable properties stemming from the file and directory names. Furthermore, we want
+to link the ficticious dat files to the Measurement records.
+
+Let's first clarify the terms we are using:
+
+.. code-block::
+
+ ExperimentalData/ <--- Category level (level 0)
+
+ 2022_ProjectA/ <--- Project level (level 1)
+
+ 2022-02-17_TestDataset/ <--- Activity / Measurement level (level 2)
+ file1.dat <--- Files on level 3
+ file2.dat
+ ...
+ ...
+
+ 2023_ProjectB/ <--- Project level (level 1)
+ ...
+
+ ...
+
+So we can see, that the three-level folder structure, described in the paper is replicated.
+We are using the term "Activity level" here, instead of the terms used in the article, as
+it can be used in a more general way.
+
+The following yaml cfood is able to match and insert / update the records accordingly:
+
+
+.. code-block:: yaml
+
+
+ ExperimentalData: # Converter for the category level
+ type: Directory
+ match: ^ExperimentalData$ # The name of the matched folder is given here!
+
+
+ subtree:
+
+ project_dir: # Converter for the project level
+ type: Directory
+ match: (?P<date>.*?)_(?P<identifier>.*)
+
+ records:
+ Project:
+ parents:
+ - Project
+ date: $date
+ identifier: $identifier
+
+
+ subtree:
+
+ measurement: # Converter for the activity / measurement level
+ type: Directory
+ match: (?P<date>[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2})(_(?P<identifier>.*))?
+
+ records:
+ Measurement:
+ date: $date
+ identifier: $identifier
+ project: $Project
+
+
+ subtree:
+
+ datFile: # Converter for the files
+ type: SimpleFile
+ match: ^(.*)\.dat$ # The file extension is matched using a regular expression.
+
+ records:
+ datFileRecord:
+ role: File
+ path: $datFile
+ file: $datFile
+ Measurement:
+ output: +$datFileRecord
+
+
+Here, we provide a detailled explanation of the specific parts of the yaml definition:
+
+.. image:: example_crawler.svg
+
diff --git a/src/doc/tutorials/example_crawler.svg b/src/doc/tutorials/example_crawler.svg
new file mode 100644
index 0000000000000000000000000000000000000000..b4e9e18f5a6e37c920bbf239eb4660898366b9b9
--- /dev/null
+++ b/src/doc/tutorials/example_crawler.svg
@@ -0,0 +1,1522 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+ width="208.60146mm"
+ height="211.33736mm"
+ viewBox="0 0 208.60146 211.33736"
+ version="1.1"
+ id="svg348"
+ xml:space="preserve"
+ inkscape:version="1.2.2 (b0a8486541, 2022-12-01)"
+ sodipodi:docname="example_crawler.svg"
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:svg="http://www.w3.org/2000/svg"><sodipodi:namedview
+ id="namedview350"
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1.0"
+ inkscape:showpageshadow="2"
+ inkscape:pageopacity="0.0"
+ inkscape:pagecheckerboard="0"
+ inkscape:deskcolor="#d1d1d1"
+ inkscape:document-units="mm"
+ showgrid="false"
+ inkscape:zoom="0.37500793"
+ inkscape:cx="286.6606"
+ inkscape:cy="503.98934"
+ inkscape:window-width="1680"
+ inkscape:window-height="981"
+ inkscape:window-x="0"
+ inkscape:window-y="32"
+ inkscape:window-maximized="1"
+ inkscape:current-layer="layer1" /><defs
+ id="defs345" /><g
+ inkscape:label="Ebene 1"
+ inkscape:groupmode="layer"
+ id="layer1"
+ transform="translate(-0.51905298,-0.01583024)"><image
+ width="180.18124"
+ height="209.55"
+ preserveAspectRatio="none"
+ xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAqkAAAMYCAYAAADox36jAAAABHNCSVQICAgIfAhkiAAAIABJREFU
+eJzs3Wd0VWX69/HvSS8nhZBiAmnEQBKEPyBFBBXpPkoRFM0EZqQ5IiOOMwqCqIgiOqA40hQGkVFk
+pKkoCAgKCCJGYKgDY0JCIIQSAuk953nBeOSYwDnpkfw+a7FWsu927eLyyr3vvbchLzfHhIjI/5w9
+e5abbrqpvsNoNHS8pSboOpIbkUFJqoiIiIg0NHb1HYCIiIiIyK8pSRURERGRBkdJqoiIiIg0OAaT
+yVStNanZ2dnXLffw8KhO9yIiIiLSCGkmVUREREQaHCWpIiIiItLgKEkVERERkQbHwZZKubm57Nmz
+h2PHjuHp6cnw4cNrOy4RERERacRsSlLt7OwICgqiqKiI9PT02o5JRERERBo5m273u7q6EhUVRVBQ
+UG3HIyIiIiKiNakiIiIi0vAoSRURERGRBkdJqoiIiIg0OEpSRURERKTBUZIqIiIiIg2OwWQymaxV
+Wr16NadPn6aoqIiioiKMRiOenp6MGjWK7Ozs67b18PCosWBFREREpHGwKUm9HiWpIiIiIlLTdLtf
+RERERBocJakiIiIi0uAoSRURERGRBkdJqoiIiIg0OEpSRURERKTBUZIqIiIiIg2OktSrZCQfZ+Xo
+nqQnHK6X8U2mskq3KSksYM24/qwZ158PHrqVtEN7aiGyK6oSn4iIiEhVONhSKTU1la1bt3LhwgWc
+nZ3p1asX0dHRNg+y78O/c/SLD3FwdTVv849qT89Jf698xLXIrYkfoV174940oM7HvnTyv8S/P5u+
+Ly6qVDsHZxeGLtwIwBcTYyuss2Zcf0oKCzDY2WP0C6TN/aMJ7nx3ncQnIiIiUhVWk1STycSOHTvo
+1asXQUFBnDhxghUrVvDkk09W6kX9rfo9SKeRE6sVbG1z8fKhy5gp9TJ2QdblWu2/93MLaNoimvPH
+/s32ORMpKSogvPs9Nrev7fhERERErmY1STUYDMTG/jJDFxERQUBAAOfOnauxr0ltm/1XvJtH0O7h
+x82/+0W2ofWgR4ArM4Qx9w3n+KaVZKWdxC+yDd3GT8fZswkAZaWlHFy9iKSdG8Bkwj+6A51HTcLR
+1R2AvIzzfP3aBHpPXUj80r9x5sBumoRE0nfaYgA2TxtL9rlTAOSmn2Xgm2vwDo4A4OyRHzm6bhkl
+hfnkXEij88iJ7FnyGp5BofR5/h2bxv9iYixdH3uBw58sIe3QDxj9m3HXX/6Gx03BFGRmsHnaWAqy
+LlGUm82acf0B8AgINsd3+VQih9b+g4uJRynKzaJZ++7c9sep2Ds6V+5AGwz4R7en0yNPs2/5XHOS
+er3+ayq+kpISOnXqRLt27Vi6dGnl4hYREZFGp9JrUsvKyrh06RK+vr41FsRtY5/jv1vWcOnkT6Tu
+30XO+VRiBoywqJO4bR09np7Ng4u+ws7BkR8/mGMuO7ByIeeO/sh9sz7m/rmf4+RmZN/yty3a519K
+59u3niWkc0+GLviS7hNmmMv6TlvM0IUbGbpwIy6ePuXiO3NgN51HT8Y3sg2HPnmP+17/iPSEw+Sm
+n7V5/N0Lp3HL/aO5f+46XL2bcmjtEuDK7O3AOWvoMnYK/tHtzXH8nAACZKelENatPwPnrGHIgi+5
+fDqR/25eXcWjDc3adSP7bArFeTlW+6+p+AoKCkhMTOTo0aNVjltEREQaj0onqbt376ZFixZ4e3tX
+qt3xTStZ8ftu5n95GRfMZS5ePnR65Bm+WziNH9+fze2Pv4TBzt6i/S33j8K1iR92Do5E3D2I1H07
+zWXHNnxEh7gncXRxA4OBNkPHcip+m0X7vIzztBk6htCufXBwccXNx9/m2L2ah+MdHIFnYCjNOnTH
+2bMJ7r6BZJ87bfP47X83gaYtonH28Cbs9n5kpibZPH5w57sJ7ngXpUWFZJ5OxDMwlAs/HbK5/a85
+uhmxd3Qm//LFGunflvZGo5Hk5GS2b99e5bhFRESk8bDpwamfJSUlsXfvXkaNGlXpgVr1G3bdNamh
+t/XihyUz8WvZFp+wVtftyzs4gsKcTAAKsy5RlJ/LrnnPW9Rx9vC0+N3BxY2bWneqdNxXMxjK/2zr
++HYOvxxqV++mlBYX2TxuXsZ5fljyGsX5eTS9OQaDnT0l/5sFrYqivBxKiwtx9fGrkf5tbe/jU36W
+WkRERKQiNiepZ86c4dNPP+V3v/sdRqOxxgM5sOpdQm7rTdrBPZz7zz4Cojtcs252WgpG/2YAOHt4
+4+jiRp8X3sHdN7DG47Kmpsa3d3SiMLvih5N2vDmR6HvjCO3aB7iy9CFlz9fl6hnsDJhKS62OdXrv
+DjyDwq7M/NrYf03El5mZiaurK05OTlZjFBERkcbNptv9p06d4uOPP2bYsGEEBNT865kyko5xYsd6
+Ov7+L9w+7gV2zXue4oI8izrJ322mtLiQorwcDqx8h8he918pMBho1fdBvl80g6L/zd4VZGaQkXSs
+xuOsUA2N7x0cweWUBHLT0670kXXJXJZzIQ2D3ZVTlZV2kuObV1XYh9EviNP7vgWTyTzT/Gvnj+1n
+7z/fpH3s+Er1X934cnNzCQsLo0ePHtc8BiIiIiI/szqTWlxczAcffIDBYOBf//oXpf+bqQsKCmL4
+8OE2D3R80yoSd3xh/t296U0MmL2SstISds6dSpcxk3F0deemWzrT/NY7iV/6N24fN+2XQJ1dWD/x
+dxTmXCb8jnstHqxqHzeBQ2v+wYZn48BgwMnNSNsHHsUnPMrm+KqjJsY3+jejQ9wENk59BHsnF9x9
+A+n13Hzs7O3pMmYyB9csYv+KeXiHRNKq7zBS9mwt10fbBx5l+5sTWfXHvgTE3Mqdf37NXLZ15hMY
+DODW9CZuG/ucxXtSbem/uvE5OTkRGhpKZGSkzcdEREREGi+DyWQyVaeD7Ozs65bXxGuqvpgYy60j
+niKwTedq9yUiIiIiDd9v6LOo1cqlRUREROQ35DeUpIqIiIhIY/GbuN0vIiIiIo1Lpd6TWhWvX/1y
+0QpMql6OLCIiIiI3IN3uFxEREZEGR0mqiIiIiDQ4v6kk1WQqq+8QatWBAwcIDAwkPj6+XsYvK7ux
+j6+IiIj8dtiUpKakpLBs2TLefPNN5syZw3fffVfbcZVz6eR/+Wr6Y3U+bk0IDw8nICCAoKAgunbt
+ymeffVZhvcDAQIYOHUrz5s3rOEI4ePAg/fr1q/NxRURERCpi04NTSUlJ9OrVi+bNm5Oens67775L
+UFAQYWFhtRzeLwqyKv5u/G/Fhg0b6NChA7t27SI2Npa8vDxiY2Mt6vj7+zNv3rx6iS89Pb1exhUR
+ERGpiE0zqXfddZd5ds/X15fg4GDy8/NrNbCfFWRmsO6poXz71rOc/89+1ozrz5px/dk8bSwAl08l
+svqPfS2WAhTlZvPxqB6UFhcCV75YdWLHer587g98PKoHX898gsKrvj1fVlrKvz9eyCdPDOCTP93H
+rvkvUJyfaxFHSUkJ7du3Z+TIkVXeF4PBQPfu3XnjjTd44YUXzNt79+5NeHg44eHhODg4cOTIEYt2
+qampdOzYkQsXLjB8+HACAgLo3bu3RWzTpk2jVatWtGzZklGjRpV7NdiKFSto3749zZo149Zbb2Xd
+unUAnD9/nrZt2xIXF8fOnTvNcVzdf2ZmJmPGjCEkJISIiAheeeUV8+dxrcV35MgRQkJCLJYSXL58
+mYCAAAoKCqp8LEVEROTGZvOaVJPJRE5ODvHx8eTn59fZN9hdvHwYOGcNXcZOwT+6PUMXbmTowo30
+nbYYAO/gCIwBzUndv8vc5uSeLQR37IG9o7N5W+K2dfR4ejYPLvoKOwdHfvxgjrnswMqFnDv6I/fN
++pj7536Ok5uRfcvftoijoKCAxMREjh49Wu196t+/PwkJCWRlZQGwZcsWkpKSSEpKws/Pr8I2Z8+e
+JS4ujsGDB3PixAmWLVtmLnvppZfYsWMHe/fu5fjx43h5eTFlyhRz+cqVK5k8eTJLly4lNTWV5cuX
+k5eXB1yZvT148CDz5s2je/fu5ji2bNlibj927FgMBgOJiYnEx8ezfv16/v73v9sUX+vWrWnRogUb
+N2401127di0DBgzAxcWlmkdSREREblQ2J6nHjh1j4cKFfPPNNwwcOBAHh1p/xarNovo/zE9frTH/
+nrRjAy3uus+izi33j8K1iR92Do5E3D2I1H07zWXHNnxEh7gncXRxA4OBNkPHcip+m0V7o9FIcnIy
+27dvr3a8np6euLi4cPbsWZvbpKamMmXKFB544AHc3d1p1qyZuWzu3Lm8+uqrGI1GDAYDU6ZMMc+U
+AsyZM4eZM2fSrl07AKKionj44YdtGvfy5cusXbuW2bNn4+joiI+PD9OnT2fx4sU2xzd+/HiL+h99
+9BEjRoywed9FRESk8bE504yOjiY6OpqMjAxWr17N7bffzi233FKbsdkspEtPflz2BvmXLoDBQPa5
+09wUc+s163sHR1CYkwlAYdYlivJz2TXveYs6zh6e5dr5+PjUSLyZmZkUFBQQFBRkcxuj0UiPHj3K
+bU9PTycrK6vcMoSrY/3pp59o3bp1lWJNSkrC19cXLy8v87abb76ZpKQkm+IDGDx4ME8//TRpaWkY
+DAZOnDjBnXfeWaV4REREpHGo9HSoj48P7dq14z//+U+dJqn2jk4UZlf88JSdvQM39xxM4rbPcXBx
+JfyOe+A6X7rKTkvB6H9lps/ZwxtHFzf6vPAO7r6B140hMzMTV1dXnJycqr4jwPr162nVqhVGo7Fa
+/QA0bdoUo9HIpk2bCAkJqbBOWFgYx48fp23bttfsx8XFhYsXL5bbHhoaSnp6OtnZ2eZP3J44caJS
+D805OjoycuRI/vnPf+Lu7k5sbCwGK18iExERkcbN6u3+/Px8Vq1aZU5gLl26xJEjRyxu59YF7+AI
+LqckkJueBkDBVQ8+AbTs8wCJ2z8n+bvNRNw1oFz75O82U1pcSFFeDgdWvkNkr/uvFBgMtOr7IN8v
+mkFRXs6VvjMzyEg6ZtE+NzeXsLCwa84W2mrXrl1MnDiR6dOnV6ufnxkMBh577DEef/xxMjOvzA6f
+P3+e/fv3m+uMGzeOKVOmcOzYlX06efIks2bNsugnJiaGw4cPk5KSAsCFCxeAK3+UDBo0iIkTJ1Ja
+WkpmZiYvvvgio0ePrlScjz76KB988AGrVq3SrX4RERGxyupMqqurK61ateKzzz7j8uXLmEwm2rVr
+x2233VYX8ZkZ/ZvRIW4CG6c+gr2TC+6+gfR6bj529vYAuPn44dW8BTnnz+DVLLxcewdnF9ZP/B2F
+OZcJv+NeYgb8kii1j5vAoTX/YMOzcWAw4ORmpO0Dj+ITHmWu4+TkRGhoaJUfGBswYAAGg4Hg4GDm
+z5/PoEGDqtRPRWbMmMHMmTO57bbbMBgMeHl5MXXqVNq3bw/A6NGjKSkpYciQIeTm5uLn58fkyZMt
++ggPD+fVV1/lzjvvxNXVlZCQENavX4+DgwNLlizhqaeeokWLFjg4OPD73/+ev/zlL5WKMSgoiOjo
+aJKTk4mKirLeQERERBo1g8lkMlWng1+/6ujXFniWX9t5tUnVG97C9+++gnfIzUTdY/lQ0BcTY7l1
+xFMEtulcY2NJ5Y0bN45bbrmF8ePH13coIiIi0sD9pj6Lej1nj8Rz9kg8LfsMvUaNmkuGpfK2bdvG
+tm3bGDt2bH2HIiIiIr8BDec9UlVUUljApxMG4ujqTrfx07FzcKzvkOQqeXl5REVF4enpyXvvvVft
+h85ERESkcbihbveLiIiIyI3hhrndLyIiIiI3DiWpIiIiItLgKEkVERERkQanUSWpJlNZldp9MTGW
+tEN7ajiaG8+BAwcIDAwkPj6+XsYvK6v8+c3LyyM8PJzw8HCcnZ3ZunVrLUR2RVXiu9rOnTvZu3dv
+DUUjIiLSsFU6SV2xYgWLFi2qjVhq1aWT/+Wr6Y/VdxjXdfvtt/PXv/613PbJkyfj6uqKn5+f+d/g
+wYPrIcLrCwwMZOjQoTRv3rzOxz548CD9+vWrdDs3NzeSkpJISkq65mdjw8PDCQgIICgoiK5du/LZ
+Z5/VWXxXO3DggPmrYSIiIje6Sr2C6sCBAxQXF9dWLLWqIOtyfYdwXUeOHMHPz4+vv/6aoqKicq9q
+euyxx5gzZ049RWcbf39/5s2bVy9jp6en12r/GzZsoEOHDuzatYvY2Fjy8vKIjY21uX114issLOS5
+555jxYoVlJaWsnnzZt566y2aNGlS5T5FREQaOptnUrOysvj222/p1q1bbcZTztkjP/L1zCfYPG0M
+a8ffy+kft7Nm3D189fIvs6KXTyXy7d8n8+mEQawcfTe75j1PaXEhAAWZGax7aijfvvUs5/+znzXj
++rNmXH82T7N8qXzSzi/5/OkHWTW2F1888xCn4rdZlBfn5bD9jaf51yN38sXEWLLPnrIoLykpoX37
+9owcObJK+7l48WIeeeQR7rnnHj755JNKtX3wwQd58cUXLX6fPXu2+fdOnTqxfPlyunfvTkBAAAMH
+DrRImkpKSpg2bRqtWrWiZcuWjBo1yuLVYqmpqXTs2JELFy4wfPhwAgIC6N27t7m8d+/e5lvmDg4O
+HDlyxFy2fft2Bg4cSK9evbj55pv54osvaNGiBf3797d5/E6dOrFv3z6GDRuGr68vnTp1IjExEYDz
+58/Ttm1b4uLi2LlzpzmOq+M7cuQIw4cPJzo6mptuuomRI0dSUFBQqWMMYDAY6N69O2+88QYvvPCC
+Tf3XRHxLlixhz549/PTTT5w+fZpu3bqRn59f6fhFRER+S2xOUj///HN69uyJs7NzbcZToTMHdtN5
+9GR8I9tw6JP3uO/1j0hPOExu+lkAstNSCOvWn4Fz1jBkwZdcPp3IfzevBsDFy4eBc9bQZewU/KPb
+M3ThRoYu3EjfaYvN/Sd/t4l9H75Ft/Ev8+Dirdzx59coKbRMYv798UJuuX80989dh6t3Uw6tXWJR
+XlBQQGJiIkePHq30/hUWFvLll19y77338vvf/57Fixdbb3SV+fPns3jxYg4dOsTGjRtJSkriqaee
+sqizbNkyVq1axalTp3BycmLixInmspdeeokdO3awd+9ejh8/jpeXF1OmTLFof/bsWeLi4hg8eDAn
+Tpxg2bJl5rItW7aYb5n7+fmVi2/z5s28/fbbdOnShddee40ffviBH374gVOnTtk8/tixY3n22Wc5
+fvw4AQEBzJw5E7gye3vw4EHmzZtH9+7dzXFs2bLF3DYhIYGHHnqIgwcPcuLECY4ePcq7775bqWN8
+tf79+5OQkEBWVpbV/msqPoPBgMlkwsHBgUcffZSgoKAqxy8iIvJbYNPt/v379+Po6EhMTAynT5+u
+7ZjK8WoejndwBJ6BoXgHR+Ds2QR330Cyz53G3fcmgjvfDUBxfi5ZZ5LxDAzlwk+HiLax/6Off0CH
+4X/GJzzqynjNwvFqFm5R59YRT9G0xZUew27vx3+/Wm1RbjQaSU5Oxs3NrdL7t2bNGvr374+TkxNR
+UVHk5OSQmJhIRESEuc7ChQt5//33zb8fOXLEnKj4+/vz5ptvMnbsWLKzs/noo4+wt7e3GGPSpEkE
+BgYC8Ic//IFHH33UXDZ37lw2btyI0WgEYMqUKXTs2JG5c+ea66SmpvLhhx/So0cPANzd3W3ev+jo
+aFq3bk1kZCQxMTH4+voSEhLCiRMnCA4Otmn8GTNm0KFDBwCGDRtWqXXRgwYNAq58eOL48eNERkby
+ww8/2Nz+1zw9PXFxceHs2bN4enpWu39r7ceMGcO///1vwsLCGDt2LJMmTcLLy6vK8YuIiPwWWE1S
+MzMz2bFjB6NHj66LeK7LYKj457yM8/yw5DWK8/NoenMMBjt7SvJybO43Ky0F7+CI69axc/jlULl6
+N6W0uKhcHR8fH5vHvNqwYcN46KGHzL/v3LkTOzvLSe5x48Zdd03qkCFDmDBhArfddhv/93//d93x
+WrduTUZGBnBlrWRWVla5ZQq/3hej0WhOUKvKcNVJ+/lnW8d3dPzlc7cBAQEUFhbaPG5qaioTJkwg
+JyeHjh074uDgQGZmZlV2Abjy30RBQYH5j4Tq9m+tvZOTE4sWLeLPf/4zr7/+Oq1atWLTpk1Wz7OI
+iMhvmdUk9fjx4xgMBt577z3gyvrB3Nxc3n77bcaOHWuldd3Y8eZEou+NI7RrHwASt60jZc/XFnXs
+HZ0ozK744SmjfxCZqck0CW1ZrTgyMzNxdXWt9PfpHRwcrvu7LV5++WWGDBnC1q1b+fbbb7njjjuu
+WTchIYHw8CszxU2bNsVoNLJp0yZCQkIqPW511dT4Li4uXLx4scKy2NhYJkyYwAMPPABcWfrw6aef
+lqtnZ2dHSUmJ1bHWr19Pq1atzDO/tvRfE/HFxMSwbNkynn76ad59910WLFhgNVYREZHfKqtrUjt3
+7syECRPM/4YNG0ZAQAATJkzA1dW1LmK0KudCGob/zTxmpZ3k+OZV5ep4B0dwOSWB3PQ0AAqyLpnL
+WvUbxr7lb5OZmvS//s5w+LP3KxVDbm4uYWFh1Z5trIr9+/ezfPlyZs2axaJFixg5ciQ5OZYzyatW
+raKgoIDMzExeeuklRo0aBVyZ0Xzsscd4/PHHzbN358+fZ//+/XUSe02NHxMTw+HDh0lJSQHgwoUL
+5rKTJ0+alz/89NNP11yPGhoaypdffonJZDLPNP/arl27mDhxItOnT69U/9WJb8KECbzzzjucOXOG
+EydO8MMPP9CyZfX+oBIREWnoboiX+XcZM5lDnyzhsz/fz/4V82nVd1i5Okb/ZnSIm8DGqY/w6YRB
+fPvWZMpKSwGI7DWEWwb9gW/+9hSr/9iPbbP+gkdA5d716eTkRGhoKJGRkTWyT7/2zjvvWLwn9ef1
+mcXFxTzyyCPMnTsXDw8P7r77bu67775yD065ubnRuXNnYmJiuP322y3KZ8yYQefOnbntttuIiYlh
+0KBBnDlzplb2oyI1MX54eDivvvoqd955J9HR0QwfPtw8Kzp37lxee+01brnlFp5//nkee6zi9+VO
+nTqVzZs3ExISwhNPPGFRNmDAAJo1a8Zf//pX5s+fz7Bhv1xjtvRfnfiefPJJ9u/fT5cuXRg0aBAj
+RowoF5+IiMiNxmAymUzV6eDqVwVVZIGn53XLJ1VveLFBp06deP311+nZs2d9hyLVMH/+fLy9vYmL
+i6vvUERERGpd5Rc/ym9SNf8WkQYgMDDQvA5WRETkRqckVeQ3YsiQIfUdgoiISJ1RktoIxMfH13cI
+IiIiIpXS4JNUa2tePTw86igSEREREakrN8TT/SIiIiJyY1GSKiIiIiINjpJUEREREWlwbFqTeuDA
+AdatW2fx/fQBAwbQunXrWgtMRERERBovm5LUgoICOnbsyD333FPb8YiIiIiI2Ha7Pz8/H3d399qO
+RUREREQEqMRManp6OitWrKCsrIzo6Gjzt+NFRERERGqaTUlq69atyc/PJywsjIsXL7Jq1SoMBgPt
+27ev7fhEREREpBGy6XZ/cHAwLVu2xMnJicDAQLp168axY8dqOzYRERERaaSq9Aoqg8GAnZ3eXiUi
+IiIitcNqppmbm8vq1au5dOkSAJcvX2bXrl1ER0fXenAiIiIi0jhZXZPq7u7OzTffzNq1a8nOzsbO
+zo4uXbrQtm3buohPRERERBohmx6cateuHe3atavtWEREREREAH0WVUREREQaICWpIiIiItLgKEkV
+ERERkQZHSaqIiIiINDhKUkVERESkwVGSKiIiIiINjpJUEREREWlwbHpPKkBCQgJbt24lOzsbLy8v
+evbsSURERG3GJiIiIiKNlE1J6pkzZ1i/fj0PP/wwAQEBXLx4kcLCwtqOTUREREQaKZuS1B07dtCz
+Z08CAgIAaNq0aa0GJSIiIiKNm01J6rlz5+jWrRvr16/nwoULBAcHc8cdd+Dk5FTb8YmIiIhII2TT
+g1PZ2dl8/fXXdOjQgYceeoiLFy+ydevW2o5NRERERBopm5JUd3d3Bg0aRGBgIK6urnTp0oWEhITa
+jk1EREREGimbklQ/Pz/S09PNvxuNxloLSERERETEpiS1S5cufPPNNxQUFGAymfjuu+9o2bJlbccm
+IiIiIo2UTQ9ORUZGkpWVxXvvvUdpaSlhYWH07NmztmMTERERkUbK5pf533rrrdx66621GYuIiIiI
+CKDPooqIiIhIA6QkVUREREQaHCWpIiIiItLg2Lwmtb54eHjUdwgiIiIiUsc0kyoiIiIiDY6SVBER
+ERFpcBpFkvrFxFjSDu2xWs9kKquDaERERETEGqtrUnNzc5k7d67FttLSUoxGI08++WStBVbXLp38
+L/Hvz6bvi4vqOxQRERGRRs9qkuru7s6zzz5rse3jjz+mTZs2tRZUfSjIulzfIYiIiIjI/1T66f4j
+R47g4OBATExMbcRTobyM83y3cBqZpxKxc3SiaYsY2v/uCTwCmgOwbGhbHn5/B84e3gDsXzGPkoI8
+Oo2caO7j4on/8O+PF5B15iR+LdvS7fGXcPZsQkFmBpunjaUg6xJFudmsGdcfAI+AYPpOW2we/+vX
+JtB76kLil/6NMwd20yQk0lxeVlrKwdWLSNq5AUwm/KM70HnUJBxd3W0qBygpKaFTp060a9eOpUuX
+1v5BFREREWnAKpWkmkwmtm3bxrBhw2orngodWPkORv9m9J4yH4BT8dssEjxbpB3YTY+/zsbZw5sd
+cybx4wdv0W38S7h4+TBwzhpOfr+F45tWXvN2f/6ldL5961la9nmArn98gaK87KviW8j5Y/u5b9bH
+ODq7Ev/+LPYtf5suYybbVA5QUFBAYmIiTk5OlT08IiIiIjecSj04lZiYiIeHB35+frUVT4XcmgZw
+7uhezh7dS1lZKcGd78bFs0ml+rjl/lG4NvHDzsGRiLsHkbrv20q1z8s4T5uhYwjt2gcHF1fcfPzN
+Zcc2fESHuCdxdHEDg4E2Q8dyKn6bzeUARqOR5ORktm/fXqm4RERERG6N0Mk7AAAgAElEQVRElZpJ
+TUhIIDw8vLZiuaa2Q8fgbPRi34dvkXkmmeCOd9Eh7kmLRLEyvIMjKMzJrFQbBxc3bmrdqdz2wqxL
+FOXnsmve8xbbnT08bSq/mo+PT6ViEhEREblRVSpJTUlJoXfv3rUVyzUZ7OyJuudhou55mMKcTPYs
+fpXd775Mr8lX3jpg5+BIQdYl85rUspLi6/aXffaUeT3rz+wdnSjMrvzDU84e3ji6uNHnhXdw9w2s
+dPnVMjMzcXV11S1/ERERafQqdbv/0qVL9fKZ0n0fvc3lU4kAOLt74tW8BZhM5nLPoFASt31OaXEh
+p3/czokdX5Tr4+TurygtLqQ4L4cDK9/h5p6DLcq9gyO4nJJAbnoaAAVZl2wLzmCgVd8H+X7RDIry
+cq60zcwgI+mYbeX/k5ubS1hYGD169LBtXBEREZEbmM0zqaWlpeTn5+Pm5lab8VTI7+Y2xC/9GzkX
+0jCVleEZFMJtY6eayzuPnMh3C6eRuG0doV370CHuyXJJoDGgOesnxVGYfYnw7v+PmAEjLMv9m9Eh
+bgIbpz6CvZML7r6B9HpuPnb29lbjax83gUNr/sGGZ+PAYMDJzUjbBx7FJzzKpnIAJycnQkNDiYyM
+rM6hEhEREbkhGEymq6YkqyA7O/u65Qs8y6+9vNqk6g0vIiIiIjegRvFZVBERERH5bVGSKiIiIiIN
+jpJUEREREWlwKv1Z1Lpmbc1rfbxtQERERERql2ZSRURERKTBUZIqIiIiIg2OklQRERERaXBsWpNa
+WlrKhg0bSEpKwmQyERUVRd++fTEYDLUdn4iIiIg0QjbNpMbHx5OTk8P48eMZN24caWlpHDlypLZj
+ExEREZFGyqaZ1Pz8fEJCQrC3t8fe3p6IiAirT92LiIiIiFSVTTOpbdu2Ze/evfz73/8mNzeXhIQE
+YmJiajs2EREREWmkbJpJ9fLyIjAwkH379vH555/TpUsXvLy8ajs2EREREWmkbEpSly9fTpcuXYiK
+iiIjI4MvvviC3bt307Vr19qOT0REREQaIau3+/Pz8zl37hxRUVEA+Pj40KdPHw4dOlTrwYmIiIhI
+42Q1SXVxccHJyYnjx49jMpkoKysjISFBt/tFREREpNZYvd1vMBiIjY1l8+bNbN68GZPJRFBQEPfe
+e29dxCciIiIijZBNa1IDAgIYMWJEbcciIiIiIgLos6giIiIi0gApSRURERGRBkdJqoiIiIg0OEpS
+RURERKTBUZIqIiIiIg2OklQRERERaXCUpNagjOTjrBzdk/SEw7XSv8lUViP9HFyzmD3/mFlue23H
+LyIiImIrm96TmpWVxRdffMGFCxdwdXWlX79+hIaG1nZsvzluTfwI7dob96YBNd73pZP/Jf792fR9
+cVGN9/2z2oxfREREpDJsmkldu3YtkZGRPPnkkwwZMoQ1a9aQnZ1d27H95rh4+dBlzBRcm/jVeN8F
+WZdrvM9fq834RURERCrD6kxqQUEBaWlp/OEPfwDA19eXDh068OOPP3L33XfXeoA14YuJscTcN5zj
+m1aSlXYSv8g2dBs/HWfPJgDkZZzn69cm0HvqQuKX/o0zB3bTJCSSvtMWA1CUl8OP78/mzIHvsLN3
+4Oaeg2kzZDQGO3sANk8bS/a5UwDkpp9l4Jtr8A6OMI9fVlrKwdWLSNq5AUwm/KM70HnUJBxd3c11
+knZ+yeFP36MgMwNXb1/+b9g4gjv1oCAzg83TxlKQdYmi3GzWjOsPgEdAsDk+awpzMvl+0QzOHv4B
+j4DmGAOa42z0Mpdbi9/a8QEoKSmhU6dOtGvXjqVLl9p+ckREREQqYNPt/uLiYoqKinB2dgbA39+f
+w4d/W+sWE7eto8fTs3H28GbHnEn8+MEcuo2fbi7Pv5TOt289S8s+D9D1jy9QlPfLTPHuhdNwdDMy
+ZMEGivNz2TpjPPbOLrQe8HsAi2Rt5eie5cY+sHIh54/t575ZH+Po7Er8+7PYt/xtuoyZDEDyd5vY
+9+Fb3D3p7/iER5GZmkRG0nHgyuzmwDlrOPn9Fo5vWlml2/27F76EvZMLD7y7meL8XLbN/qtFkmot
+fmvHB678MZOYmIiTk1Ol4xMRERH5Nau3+11cXAgMDGTPnj0UFhaSmJjI1q1bycnJqYv4aswt94/C
+tYkfdg6ORNw9iNR9Oy3K8zLO02boGEK79sHBxRU3H38AinKzOfn9Fjr+4Wns7B1wNnrR7uHx/PTV
+GpvHPrbhIzrEPYmjixsYDLQZOpZT8dvM5Uc//4AOw/+MT3gUAF7Nwgnv3r/6Ow0U5WSRsudruox+
+FntHJ1w8mxD0f10r3c+1js/PjEYjycnJbN++vUbiFhERkcbNppnUBx98kG+++YaPPvqI5s2bc8cd
+d5CQkFDbsdUa7+AICnMyLbY5uLhxU+tO5ermnE/FxbMJTm5G8zbPwBByzqfaNFZh1iWK8nPZNe95
+i+3OHp7mn7PSUixur9ek7POpuHg1wcnoab3ydVzr+FzNx8enWmOIiIiI/MymJNXb25v777/f/Pum
+TZsICPjtPgGenZaC0b+ZTXXd/QIpyLpEcX6ueQ1p9rnTuPsF2dTe2cMbRxc3+rzwDu6+gRXWMfoH
+kZmaTJPQltfsx97RicLsyj885erlQ2F2JqXFhdg7Ole6fWVkZmbi6uqqW/4iIiJSbTY93Z+cnExh
+YSEAJ06c4PDhw3Ts2LFWA6tpyd9tprS4kKK8HA6sfIfIXvdbbwQ4G70I6dyTvf+cg6mslKK8HP79
+rwVE9h5i28AGA636Psj3i2ZQlHdliURBZgYZScfMVVr1G8a+5W+TmZoEQM6FMxz+7H2LbryDI7ic
+kkBuetqVPrIu2TS8W9MAmoS15OCqRWAykX02haQdG2yLvRJyc3MJCwujR48eNd63iIiIND42zaSe
+O3eO9evXU1RUhI+PDyNGjMDV1bW2Y6tRDs4urJ/4OwpzLhN+x73EDBhhc9vbH3+J+KV/Y824/4ed
+vT0RPQbQuhLt28dN4NCaf7Dh2TgwGHByM9L2gUfNa1Ajew3BVFrKN397ipKCfFy8mtBmyBiLPoz+
+zegQN4GNUx/B3skFd99Aej03Hzt7e6vj3/nU63w3/0VWPdobn7AoWvS4j7yL522O3xZOTk6EhoYS
+GRlZo/2KiIhI42QwmUym6nRg7X2pCzyvvxZykpXhrfXv4eFx3XK48gqqW0c8RWCbzlbrVkdZaSkr
+RnRl0Fuf2LycQERERETKs2km9cZQrVz8unLOn8HoH8TZwz/g4OyKWx1+senCfw+y5ZVxFZY9/P4O
+87tcRURERH5LGlGSWjvyLp7j27cnk59xAXtnF+54ciZ29nV3WP1atiX2n7vqbDwRERGRutAobveL
+iIiIyG/LDT+TqiRXRERE5LfHpldQiYiIiIjUJSWpIiIiItLgKEmtBJOprL5DqFUHDhwgMDCQ+Pj4
+ehm/rOzGPr4iIiJiOzu48rWgr7/+mgULFvDhhx+Wq1RWVsamTZt4++23mT9/Pnv37q3zQOvbpZP/
+5avpj9V3GFUSHh5OQEAAQUFBdO3alc8++6zCeoGBgQwdOpTmzZvXcYRw8OBB+vXrV+fjioiISMPk
+AGBnZ0dQUBBFRUWkp6eXq/Tdd9+RlZXFn/70JwoLC3n//ffx8fEhPDy8zgOuLwVZl+s7hGrZsGED
+HTp0YNeuXcTGxpKXl0dsbKxFHX9/f+bNm1cv8VV03YmIiEjjZQfg6upKVFQUQUFBFVbat28fPXr0
+wM7ODldXV26//Xb27dtXp4HWl4LMDNY9NZRv33qW8//Zz5px/Vkzrj+bp40F4PKpRFb/sa/FUoCi
+3Gw+HtWD0uJC4MoXr07sWM+Xz/2Bj0f14OuZT1CYdclcv6y0lH9/vJBPnhjAJ3+6j13zX6A4P9ci
+jpKSEtq3b8/IkSOrvC8Gg4Hu3bvzxhtv8MILL5i39+7dm/DwcMLDw3FwcODIkSMW7VJTU+nYsSMX
+Llxg+PDhBAQE0Lt3b4vYpk2bRqtWrWjZsiWjRo0q91aFFStW0L59e5o1a8att97KunXrADh//jxt
+27YlLi6OnTt3muO4uv/MzEzGjBlDSEgIERERvPLKK5SWltoU35EjRwgJCbFYSnD58mUCAgIoKCio
+8rEUERGR2mV1TWpZWRlZWVn4+vqya9cujh07hr+/PxkZGXURX71z8fJh4Jw1dBk7Bf/o9gxduJGh
+CzfSd9piALyDIzAGNCd1/y8v1D+5ZwvBHXtg7+hs3pa4bR09np7Ng4u+ws7BkR8/mGMuO7ByIeeO
+/sh9sz7m/rmf4+RmZN/yty3iKCgoIDExkaNHj1Z7n/r3709CQgJZWVkAbNmyhaSkJJKSkvDz86uw
+zdmzZ4mLi2Pw4MGcOHGCZcuWmcteeuklduzYwd69ezl+/DheXl5MmTLFXL5y5UomT57M0qVLSU1N
+Zfny5eTl5QFXZm8PHjzIvHnz6N69uzmOLVu2mNuPHTsWg8FAYmIi8fHxrF+/nr///e82xde6dWta
+tGjBxo0bzXXXrl3LgAEDcHFxqeaRFBERkdpiNUktKSnBzs4Og8FAcnIyaWlpODo6UlhYWBfx/SZE
+9X+Yn75aY/49accGWtx1n0WdW+4fhWsTP+wcHIm4exCp+3aay45t+IgOcU/i6OIGBgNtho7lVPw2
+i/ZGo5Hk5GS2b99e7Xg9PT1xcXHh7NmzNrdJTU1lypQpPPDAA7i7u9OsWTNz2dy5c3n11VcxGo0Y
+DAamTJlinikFmDNnDjNnzqRdu3YAREVF8fDDD9s07uXLl1m7di2zZ8/G0dERHx8fpk+fzuLFi22O
+b/z48Rb1P/roI0aMGGHzvouIiEjds/oyfycnJ+BKshoXFwdASkqKXoJ/lZAuPflx2RvkX7oABgPZ
+505zU8yt16zvHRxBYU4mAIVZlyjKz2XXvOct6jh7lP9Sl4+PT43Em5mZSUFBwTWXd1TEaDTSo0eP
+ctvT09PJysoqtwzh6lh/+uknWrduXaVYk5KS8PX1xcvLy7zt5ptvJikpyab4AAYPHszTTz9NWloa
+BoOBEydOcOedd1YpHhEREakbNn1xyt/fn9TUVEJDQwE4ffo0/v7+tRpYQ2Pv6ERhdsUPT9nZO3Bz
+z8EkbvscBxdXwu+4BwyGa/aVnZaC0f/KTJ+zhzeOLm70eeEd3H0DrxtDZmYmrq6u5j8cqmr9+vW0
+atUKo9FYrX4AmjZtitFoZNOmTYSEhFRYJywsjOPHj9O2bdtr9uPi4sLFixfLbQ8NDSU9PZ3s7Gzz
+H0YnTpwgLCzM5hgdHR0ZOXIk//znP3F3dyc2NhbDdc6PiIiI1D+b3pPasWNHtm/fTmlpKTk5OcTH
+x9OhQ4fajq1B8Q6O4HJKArnpaQAUXPXgE0DLPg+QuP1zkr/bTMRdA8q1T/5uM6XFhRTl5XBg5TtE
+9rr/SoHBQKu+D/L9ohkU5eVc6Tszg4ykYxbtc3NzCQsLu+Zsoa127drFxIkTmT59erX6+ZnBYOCx
+xx7j8ccfJzPzyuzw+fPn2b9/v7nOuHHjmDJlCseOXdmnkydPMmvWLIt+YmJiOHz4MCkpKQBcuHAB
+uDIjO2jQICZOnEhpaSmZmZm8+OKLjB49ulJxPvroo3zwwQesWrVKt/pFRER+AxwAVq9ezenTpykq
+KqKoqIi33noLT09PRo0aBUC7du24fPkyCxcuxM7Ojt69exMQEFCvgdc1o38zOsRNYOPUR7B3csHd
+N5Bez83Hzt4eADcfP7yatyDn/Bm8mpV/NZeDswvrJ/6OwpzLhN9xLzEDfkmU2sdN4NCaf7Dh2Tgw
+GHByM9L2gUfxCY8y13FyciI0NJTIyMgqxT9gwAAMBgPBwcHMnz+fQYMGVamfisyYMYOZM2dy2223
+YTAY8PLyYurUqbRv3x6A0aNHU1JSwpAhQ8jNzcXPz4/Jkydb9BEeHs6rr77KnXfeiaurKyEhIaxf
+vx4HBweWLFnCU089RYsWLXBwcOD3v/89f/nLXyoVY1BQENHR0SQnJxMVFWW9gYiIiNQrg8lkMlWn
+g1+/aujXFniWX1t5tUlWhrfWv7W1sdVtXxnfv/sK3iE3E3WP5UNBX0yM5dYRTxHYpnONjSWVN27c
+OG655RbGjx9f36GIiIiIFfosag05eySes0fiadln6DVqVOtvAammbdu2sW3bNsaOHVvfoYiIiIgN
+bHpwSq6tpLCATycMxNHVnW7jp2Pn4FjfIclV8vLyiIqKwtPTk/fee6/aD52JiIhI3dDtfr1KS0RE
+RKTB0e1+EREREWlwlKSKiIiISIOjJFVEREREGhwlqXXIZCqrUrsvJsaSdmhPDUdTseLiYiZOnEhe
+Xl6djFeRGTNm8MQTT1S6XVlZ1Y5vfTt27BhvvvlmjfRVn+fvwIEDBAYGEh8ff916VT2/1lg7/9bi
+S0lJoU+fPtx00020a9eOTZs2Vap9Q1WT15eISF2ygytfM/r6669ZsGABH374YblK1srFuksn/8tX
+0x+r7zCsGj58OE2bNsXNzc287dy5c8TGxhIaGkpwcDCLFi2yaHP//ffj6+tLWFgYoaGh9O3bl8OH
+D9dp3AcPHqRfv351OqatrB2/m2++md27dzNnzpxqj1Wf5y8wMJChQ4fSvHnzau9HZdly/q3F98wz
+z9CqVSuSk5PZvXs3d9xxR6XaN1Q1eX2JiNQlOwA7OzuCgoJo0aJFxZWslIt1BVmX6zsEq/71r39R
+UFDApEmTLLafOXOGBx54gOTkZD7//HP+9Kc/cebMGYs6r7/+OsnJySQnJzNo0CBiY2OvOc7SpUtZ
+sWJFjcaenp5e5bYzZsxg27ZtNRfMr1g7fg4ODixbtoxFixaZPx1bFfV9/vz9/Zk3bx6BgYFV3oeq
+suX8W4vv4MGDDB8+HBcXF1xdXS0SfVvaV1VtX381dX2JiNQ1OwBXV1eioqIICgqqsJK18hvZ2SM/
+8vXMJ9g8bQxrx9/L6R+3s2bcPXz18i+zopdPJfLt3yfz6YRBrBx9N7vmPU9pcSEABZkZrHtqKN++
+9Szn/7OfNeP6s2ZcfzZPs3ypfNLOL/n86QdZNbYXXzzzEKfit1mUF+flsP2Np/nXI3fyxcRYss+e
+sigvKSmhffv2jBw5ssr7+sYbbzBz5sxy29u3b8/QoUMxGAy0bNkSDw+Pa75v1GAwMHToUI4dO1bu
+9mtaWhr33XcfW7dupX///ubtGRkZPPzwwwQEBNC1a1eOHj1q0e7IkSMMHz6c6OhobrrpJkaOHElB
+QQEA58+fp23btsTFxbFz507Cw8MJDw+nd+/eFsdm2rRptGrVipYtWzJq1CiLV5MNGzaM559/nqee
+eor8/PzKH7j/efPNN9m9e3e57bYcPzc3N5555hnmzZtX5fHr6/z17t3bfNwdHBw4cuSIRTtr59fa
++enUqRP79u1j2LBh+Pr60qlTJxITEwHbzr+1+J555hmioqJISEhg6NChlW5vLf7U1FQ6duzIhQsX
+GD58OAEBARb91/b1BzVzfYmI1DWtSbXBmQO76Tx6Mr6RbTj0yXvc9/pHpCccJjf9LADZaSmEdevP
+wDlrGLLgSy6fTuS/m1cD4OLlw8A5a+gydgr+0e0ZunAjQxdupO+0xeb+k7/bxL4P36Lb+Jd5cPFW
+7vjza5QUFljE8O+PF3LL/aO5f+46XL2bcmjtEovygoICEhMTyyUANu/jmTNkZWURExNzzTplZWWM
+Hj2aJ554Al9f32vWee+99+jcuTN2dr9cXv/617/o1asXY8aM4cMPP6RJkybmsrFjx+Lo6EhKSgrr
+1q0jNTXVos+EhAQeeughDh48yIkTJzh69CjvvvsucGV26+DBg8ybN4/u3buTlJREUlISW7ZsMbd/
+6aWX2LFjB3v37uX48eN4eXkxZcoUc3lkZCTbt28nKCiIrl27smdP5db//jwrmZubS3Z2NmVlZZw7
+d67Sx2/w4MF8+umnlRr76hjq6/xt2bLFfNz9/PzK9Wnt/Fo7Pz/38eyzz3L8+HECAgLMybgt599a
+fLNmzeLYsWOEhYWxYcOGSre3Jf6zZ88SFxfH4MGDOXHiBMuWLTOX1dX1V53rS0SkPihJtYFX83C8
+gyPwDAylWYfuOHs2wd03kOxzpwEI7nw3wR3vorSokMzTiXgGhnLhp0M293/08w/oMPzP+IRHXRmv
+WTjh3ftb1Ll1xFM0bRGNs4c3Ybf3IzM1yaLcaDSSnJzM9u3bq7SPycnJREREXLfO9OnT8fT0ZNq0
+aeXKJk2aRFhYGOHh4fz444989NFH5rIlS5Ywe/ZsduzYweDBgy3aXbp0iU8//ZS3334bZ2dn/Pz8
+6NOnj0WdQYMGMWDAAAoKCjh69CiRkZH88MMPNu/b3LlzefXVVzEajRgMBqZMmcK6dess6tjZ2fHM
+M8+wfPlyBg4cyMGDB23uf+nSpXTt2pW1a9fy6quvcuedd/Ltt9+Wq3e94wfg4+NDXl4eRUVFNo/9
+s/o6f9bYcn5tOT8zZsygQ4cONG3alGHDhjWo29a2xJ+amsqUKVN44IEHcHd3p1mzZhbldXH9Vef6
+EhGpD/osaiUYDBX/nJdxnh+WvEZxfh5Nb47BYGdPSV6Ozf1mpaXgHXz9BMPO4ZdT5erdlNLi8v+j
+8fHxsXnMXysqKsLR8fqfdD1y5Agvv/xyhWWvv/46o0ePrrCsT58+LF++nKlTpzJ79myMRqO57OfZ
+qatn5n4tNTWVCRMmkJOTQ8eOHXFwcCAzM9OGvbqyVjErK6vcMoiKjlVKSgpPPvkkAwcOtJrwXe25
+557jkUceoUOHDhQXF3PgwAHs7e3L1bve8fuZg4MDRUVFlf58a32dP2usnV9bz8/V+xYQEEBhYaHN
+MdQmW+M3Go306NHjun3V9vUHVb++RETqg2ZSa8CONycS3v0e+rzwDh1+N4HANp3L1bF3dKIwu+KH
+p4z+QWSmJlc7jszMzCrPkjRv3pxTp05dt86qVauIioqqdN8hISFs3bqVmJgYunTpwjfffGMu8/f3
+JyMjw7zGtCKxsbHExsayadMmZsyYwd13312ujouLCxcvXiy3vWnTphiNRjZt2sR//vMf87/9+/db
+1FuyZAn9+vXjr3/9K4sXL8bd3b1cX9c7vlOnTmXx4sUMGjSIJUuWVFjH2vErKCigrKysUkngz+rr
+/Flj7fzaen6sudb5r201FX9dXH/Vub5EROqDktQakHMhDcP/1u9lpZ3k+OZV5ep4B0dwOSWB3PQ0
+AAqyLpnLWvUbxr7lb5tv4edcOMPhz96vVAy5ubmEhYVZna25loiICLKysjh79myF5RkZGYSGhvLO
+O+9UqX+DwcCECRP49NNPeeGFF8xr45o3b07btm155ZVXMJlMJCQkWNxqBjh58qR5Zuinn34yr0e9
+WkxMDIcPHyYlJQWACxcumMd97LHHePzxx82zr+fPn7dIIp5//nm2bdvG7t27ueeeeyqM/3rHNzs7
+m5iYGAYOHMj06dNJS0srV8eW47d582aLB2oqo77OnzXWzq8t58cW1zr/ta0m4q+L6w+qd32JiNQH
+B4DVq1dz+vRpioqKKCoq4q233sLT05NRo0ZhS3lj12XMZA6uWcT+FfPwDomkVd9hpOzZalHH6N+M
+DnET2Dj1EeydXHD3DaTXc/Oxs7cnstcQTKWlfPO3pygpyMfFqwlthoypVAxOTk6EhoYSGRlZpX0w
+GAw8+uijzJo1izfeeKNcuclkqlK/v/bzQyJXJ1MrVqxg9OjRNG/enHbt2jFixAiLh2vmzp3LjBkz
+eP7557nlllt47LHH+OSTTyz6DQ8PN6/Hc3V1JSQkhPXr1+Pg4MCMGTOYOXMmt912GwaDAS8vL6ZO
+nUr79u0B+OMf/2j13ZfXO74eHh4888wzANjb2/Piiy+Wq2Pt+JlMJmbNmmV1OcC11Of5s8ba+bV2
+fmxxvfNf26obf11df9W5vkRE6oPBVM3/e139qpWKLPD0vG75JCvDW+vfw8OjVts3JsXFxXTr1o2X
+X365wb4Y/0b1yiuvcPLkSRYvXmy98jXo/Mm11MT1JSJS13S7X8wcHR35/PPPeeutt+r1s6iNzeHD
+hzly5AgLFiwoV/b999/TpEmTCv+VlpZa1NX5k4pc7/oSEWnINJOqmVQRERGRBkczqSIiIiLS4ChJ
+FREREZEGR0mqiIiIiDQ4SlJFREREpMFRkioiIiIiDY4DXPmayZ49ezh27Bienp4MHz7colJqaipb
+t27lwoULODs706tXL6Kjo+slYBERERG58TkA2NnZERQURFFREenp6RYVTCYTO3bsoFevXgQFBXHi
+xAlWrFjBk08+qdc3iYiIiEitcABwdXUlKiqqwiTVYDAQGxtr/j0iIoKAgADOnTunJFVEREREakWl
+16SWlZVx6dIlfH19ayMeEREREZHKJ6m7d++mRYsWeHt710Y8IiIiIiKVS1KTkpLYu3cv/fv3r614
+RERERERsT1LPnDnDp59+ykMPPYTRaKzNmERERESkkXOwpdKpU6dYvXo1w4YNIyAgoLZjEhEREZFG
+zmAymUyrV6/m9OnTFBUVUVRUhNFoxNPTk1GjRlFcXMysWbMwGAw4OTlRWloKQFBQEMOHDyc7O/u6
+Ayzw9Lxu+SST6brl1vq39oaB6rYXERERkbpnMJmsZIlWKEkVERERkZqmz6KKiIiISIOjJFVERERE
+GhwlqSIiIiLS4ChJFREREZEGR0mqiIiIiDQ4SlJFREREpMFRknoDOrhmMXv+MbO+wxARERGpMgeA
+3Nxc9uzZw7Fjx/D09GT48OEWlVJSUvjmm2+4ePEiBoOBLl26cPvtt9dLwCIiIiJy43MAsLOzIygo
+iKKiItLT08tVSkpKolevXjRv3pz09HTeffddgoKCCAsLq+t4RURERKQRcABwdXUlKirqmknqXXfd
+Zf7Z19eX4OBg8vPz6y7KepaXcZ6vX5tA76kLiV/6N84c2E2TkI3uwLwAACAASURBVEj6TlsMQFlp
+KQdXLyJp5wYwmfCP7kDnUZNwdHU395G080sOf/oeBZkZuHr78n/DxhHcqQcARXk5/Pj+bM4c+A47
+ewdu7jmYNkNGY7Czt2n8wpxMvl80g7OHf8AjoDnGgOY4G70s4v9u4TQyTyVi5+hE0xYxtP/dE3gE
+NDfXKSkp+f/s3XlYlXX+//HngcOBA4cDokKAsqaipqG5ZFmZ29S3bZJyMmx+E2rTMuE002iaU7bY
+PmWuZaU1TpKm5WiaWpbaYkbqYGo4iSiKCyBw2DkI9+8Pp1MnF0DZ1NfjurwuuD/b6z6Qvb1Xevfu
+TXx8PPPmzWvsj1RERETktMx17WgYBqWlpfzwww+Ul5fToUOHxszV4pQX5PHF1EfoOOQ2+v3xMZxl
+P79uNW3RbHLSt3Ljiwvx8raS+vaLbHl3Gn1HTwBg79er2fKvqVw7/lWCouNwZGeSn7nLNX7j7Ml4
++doYNmslVeWlrJ3yAJ7ePnS96fd1Wn/j7CfwtPhw2+trqCovZd1Lf3UrUtMWvYYtOJzBE2cCsD91
+nVsBDVBRUUFGRgYWi6VhPzgRERGRM1DnG6fS09OZPXs2n3/+OTfffDNmc53r2/NCWX4O3RJGE9lv
+CGYfK75Bwa629JUL6Jk4Fi8fXzCZ6JYwhv2p61ztO5fPp+fIPxMUHQdAQHg00f2vA8BZWsy+bz6l
+1/97GA9PM962AOLveIAfP1lSp/WdJUVkbfqMvqMewdPLgo+9FWGX9nMb69s6hCM7N3N452Zqaqpp
+3+dafOyt3PrYbDb27t3L+vXrG+wzExERETlTda40O3fuTOfOncnPz2fx4sVcccUVXHLJJY2ZrUUx
++/hyUdfeJ2yvLCrAWV7KVzP+7rbd29/u+rroUBaB7WNPOm9JTjY+9lZYfG2ubfbQCEpysuu0fnFO
+Nj4BrbDY7Ce0/aR7wmi8bQFs+ddUHAf30r7XNfRMHOtWaAMEBQWdcg4RERGRplTvw6FBQUHEx8fz
+ww8/XFBF6ql4+wfi5ePLkMdew69N6En72ILDcGTvpVVkxxPa/NqGUlFUQFV5qesUfPGRA/i1DavT
++taAICqLHVRXVeLp5X3SPiYPT+Kuv4O46++gssTBpjeeYePrTzFownS3fg6HA6vVqlP+IiIi0uxq
+Pd1fXl7O+++/z9GjRwEoKChgx44dhIeHN3q4c4LJRKeht/PNnCk4y0oAqHDkk5+Z7urS6TfD2fLu
+NBzZmQCU5B5k+7/fBsDbFkBEn4Fs/ucrGDXVOMtK+M97s+gweFidlvdtHUKrqI5se38OGAbFh7PI
+3LDSrc+WBdMo3J9xfD0/OwHtYsAw3PqUlpYSFRXFgAEDzuRTEBEREWlQZoDFixdz4MABnE4nTqeT
+qVOnYrfbSUpKwmq10qlTJ/79739TWFiIYRjEx8dz+eWXN3f2FqNHYjLfL3mTlY8kgsmExddG99vu
+cV2D2mHQMIzqaj5/4SGOVZTjE9CKbsNGu8Zfcf8TpM57gSX3/R8enp7EDriJrjfdVef1r37oeb6e
++Tjv3zOYoKg4YgbcSNnRHFd724u7kTrvBUpyD2HU1GAPi+DyMZPc5rBYLERGRl5wN8SJiIhIy2Qy
+jF8dUqun4uLi07bPsp/6WkmA8bUsX9v8/v7+jTpeRERERJqeXosqIiIiIi2OilQRERERaXFUpIqI
+iIhIi3NhPZH/DOiaVhEREZGmpyOpIiIiItLiqEgVERERkRZHRep5JH/vLhaNGkje7u2NMr9h1DTK
+vCIiIiK/ZobjbxvatGkT6enp2O12Ro4cecoBKSkpFBcXc8899zRZyHPd58//mSM/bMHLx4phQEB4
+FL3/8DcCIy5u0HV8W7Ulst9g/FqHNOi8AAX7/kvq2y8x9PE5DT63iIiIyK+ZATw8PAgLC8PpdJKX
+l3fKzmlpaVRVVTVZuPPJZXf9mQ6DhoFhkL5qIetfGcctr3zQoGv4BATRd/TEBp3zJxVFhY0yr4iI
+iMjJmAGsVitxcXGnLVKLior44osvuP7661m7dm2ThjyvmExE9hvMt/OexzBqMJk8KMvP4bPnkhk8
+aTap817gYNpGWkV0YOjkNwBwlpXw3dsvcTDtazw8zVw88Ld0GzYKk4cnAGsmj6H4yH4ASvMOc/PL
+SwhsH+tasqa6mm2L55D55UowDII796RP0ni8rH6uPplffsz2pXOpcORjDWzDpcPvo33vAVQ48lkz
+eQwVRQU4S4tZct91APiHtHflAzh27Bi9e/cmPj6eefPmNfrHKCIiIue3Oj+Cavny5QwcOBBvb+/G
+zHPeM4wadq/9kLYXd8Nk+vmS4PKCPL6Y+ggdh9xGvz8+hrPs50dfbZw9GS9fG8NmraSqvJS1Ux7A
+09uHrjf9HsCtWFw0auAJa6Ytmk1O+lZufHEhXt5WUt9+kS3vTqPv6AkA7P16NVv+NZVrx79KUHQc
+juxM8jN3AcePzt78yhL2ffMpu1YvOuXp/oqKCjIyMrBYLGf/IYmIiMgFr043Tm3duhUvLy+6dOnS
+2HnOW5vnT2XJvb9hyb3Xk5exk6sees6tvSw/h24Jo4nsNwSzjxXfoGAAnKXF7PvmU3r9v4fx8DTj
+bQsg/o4H+PGTJXVeO33lAnomjsXLxxdMJroljGF/6jpX+87l8+k58s8ERccBEBAeTXT/6+q1fzab
+jb1797J+/fp6jRMRERE5mVqPpDocDjZs2MCoUaOaIs95y3VN6imYfXy5qGvvE7aX5GTjY2+Fxdfm
+2mYPjaAkJ7tO61YWFeAsL+WrGX932+7tb3d9XXQoy+3ygDMVFBR01nOIiIiIQB2K1F27dmEymZg7
+dy5w/NrD0tJSpk2bxpgxYxo94IXOr20oFUUFVJWXuq4hLT5yAL+2YXUa7+0fiJePL0Meew2/NqEn
+7WMLDsORvZdWkR1POY+nl4XK4tPfPOVwOLBarTrlLyIiImet1tP9ffr0ITk52fVn+PDhhISEkJyc
+jNVqbYqMFzRvWwARfQay+Z+vYNRU4ywr4T/vzaLD4FMflXVjMtFp6O18M2cKzrISACoc+eRnpru6
+dPrNcLa8Ow1HdiYAJbkH2f7vt92mCWwfS2HWbkrzDh2fo6jArb20tJSoqCgGDBhwZjsqIiIi8gtm
+gMWLF3PgwAGcTidOp5OpU6dit9tJSkpq7nwCXHH/E6TOe4El9/0fHp6exA64ia433VXn8T0Sk/l+
+yZusfCQRTCYsvja633aP6xrUDoOGYVRX8/kLD3GsohyfgFZ0GzbabQ5bcDg9E5NZNekPeFp88GsT
+yqBHZ+LhefwJAxaLhcjISDp06NBwOy4iIiIXLJNhGMbZTFBcXHza9ll2+2nbx9eyfG3z+/v7t+jx
+TammupqUu/pxy9QPsQWHN3ccERERkTOm16KeB0pyDgJwePu3mL2t+DbCG6dEREREmlKdn5MqLVPZ
+0SN8MW0C5fm5eHr7cNXYZ/Hw1I9VREREzm2qZs5xvq1DuP7pd5o7hoiIiEiDUpHayM6la1pFRERE
+WgpdkyoiIiIiLY6KVBERERFpcVSkSoMxjJrmjiAiIiLnCTMcf1vQpk2bSE9Px263M3LkSLdOaWlp
+LFu2DC8vL9e2m266ia5duzZtWmmxCvb9l9S3X2Lo43OaO4qIiIicB8wAHh4ehIWF4XQ6ycvLO6FT
+RUUFvXr14vrrr2/ygHJuqCgqbO4IIiIich4xA1itVuLi4k5ZpJaXl+Pn59fk4eS4j8aNoMuNI9m1
+ehFFh/bRtkM3rnzgSbztrQAo3J/B9x+8ydGMnThLiwjv0Z/L/zgJTy9vAMryc/jsuWQGT5pN6rwX
+OJi2kVYRHRg6+Q3g+Juqti2eQ+aXK8EwCO7ckz5J4/Gy+rnW73fvY2z/8C0Off8ttuBwrvnLC/hf
+1J4KRz5rJo+hoqgAZ2kxS+67DgD/kPau+cvyc/h69mQc+zPw8LLQOqYLPe58EP+Qdq59PHbsGL17
+9yY+Pp558+Y12WcrIiIiLVOdrkmtqKggKyuLlJQU3n33XbZs2dLYueRXMtYtY8DDL3H7nE/wMHvx
+3fxXXG3Fh7KIuvI6bn5lCcNmfUzhgQz+u2ax2/jygjy+mPoIEX0GkjDrY/onT3G1pS2azZGd33Hj
+iwu5dfpyLL42trw7zW38xtmTueTWUdw6fRnWwNZ8/8FbAPgEBHHzK0voO2YiwZ17kDB7FQmzV7kK
+1OPzv4YtOJyE2au4ddoyoq+8zlUA/6SiooKMjAx27tzZYJ+ZiIiInLvqVKR27dqVPn36kJCQwMCB
+A/nyyy/ZunVrY2eTX7jk1iSsrdriYfYi9tpbyN7ypautfZ9rad/rGqqdlTgOZGAPjST3x+/dxpfl
+59AtYTSR/YZg9rHiGxTsaktfuYCeiWPx8vEFk4luCWPYn7rObXyPO5NpHdMZb/9Aoq74DY7szDpn
+920dwpGdmzm8czM1NdW073MtPv87CvwTm83G3r17Wb9+fT0+FRERETlf1elh/u3bt3d9HRoaypVX
+Xkl6ejo9evRotGByaoHtY6kscbi+L8vP4du3nqOqvIzWF3fB5OHJsbIStzFmH18u6tr7hLkqiwpw
+lpfy1Yy/u2339re7fe9h/vlXxRrYmuoqZ53zdk8YjbctgC3/morj4F7a97qGnolj3QplgKCgoDrP
+KSIiIue3M3rjlMlkwsNDT69qLsWHsrAFh7u+3/DyODrfkEhkvyHA8UsDsjZ9Vqe5vP0D8fLxZchj
+r+HXJvSMM3l6WagsPvnNUyYPT+Kuv4O46++gssTBpjeeYePrTzFownS3fg6HA6vVisViOeMcIiIi
+cn6otdIsLS1l8eLFFBQUAFBYWMhXX31F586dGz2c/Gzv12uorqrEWVZC2qLX6DDoVldbSe4hTP/7
+R0PRoX3sWvN+3Sc2meg09Ha+mTMF5/+OvlY48snPTK9XvsD2sRRm7aY079DxOYoKXG1bFkyjcH8G
+AN5+dgLaxYBhuI0vLS0lKiqKAQMG1GtdEREROT+ZARYvXsyBAwdwOp04nU6mTp2K3W4nKSkJPz8/
+Lr74Yj744AOKi4vx8PCgb9++dO/evbmzX1DM3j6sGHcnlSWFRF91A11uusvV1nf0BLYtmcPWlBkE
+RnSg09DhZG1aW+e5eyQm8/2SN1n5SCKYTFh8bXS/7R6CouPqPIctOJyeicmsmvQHPC0++LUJZdCj
+M/Hw9KTtxd1InfcCJbmHMGpqsIdFcPmYSW7jLRYLkZGRdOjQoc5rioiIyPnLZBi/OqRVT8XFxadt
+n2W3n7Z9fC3L1za/v7//eT0ejj8C6rK7HiK0W59a+4qIiIicD3Rh6TnjrP4tISIiInJOUZEqIiIi
+Ii3OGd3dL03rxhdSmjuCiIiISJPSkVQRERERaXFUpIqIiIhIi6MiVURERERaHBWp0mDS0tIIDQ0l
+NTW1UeavqalplHlFRESk5THD8bf9bNq0ifT0dOx2OyNHjjyh4+7du1m7di3FxcUEBAQwcOBAYmNj
+mzyw1N+tt97KF198gc1mwzAMOnXqxMsvv8wll1zSoOuEhoaSkJBAu3btGnRegG3btvHXv/6VTz75
+pMHnFhERkZbHDODh4UFYWBhOp5O8vLwTOh08eJAVK1Zwxx13EBISwtGjR6msrGzysHLmnn/+eUaN
+GoVhGMyaNYsRI0bw/fffN+gawcHBzJgxo0Hn/MnJfi9FRETk/OUBYLVaiYuLIyws7KSdNmzYwMCB
+AwkJCQGgdevWp+wrLZvJZCIhIYH09HTX6fPs7Gx69epFbm4uI0eOJCQkhMGDB7vGOBwORo8eTURE
+BLGxsTz99NNUV1e72gcPHkx0dDTR0dGYzWZ27NjhtuaxY8eYPHkynTp1omPHjiQlJZ3wJq6UlBR6
+9OhBeHg4l112GcuWLQMgJyeH7t27k5iYyJdffula55f5RERE5PxTp2tSjxw5QmBgICtWrODtt99m
+7dq1OJ3Oxs4mjaCmpoa5c+fSp08fPDx+/vEfPnyYxMREfvvb37Jnzx7eeecdV9uYMWMwmUxkZGSQ
+mprKihUrePXVV13tn376KZmZmWRmZtK2bdsT1nziiSfYsGEDmzdvZteuXQQEBDBx4kRX+6JFi5gw
+YQLz5s0jOzubd999l7KyMuD40dlt27YxY8YM+vfv71rn008/bYyPR0RERFqIOj3Mv7i4mM8++4yh
+Q4cSGBjI8uXLWbt2Lddff31j5xPguuuu4+jRo27brrnmGl566aU6zzF+/HieeuopDMPgsssuY8GC
+BW7t2dnZ/Otf/2LAgAEA+Pn5AVBYWMgHH3zA0aNH8fLyIigoiCeffJLk5GT+8pe/1Gnt6dOns2rV
+Kmw2GwATJ06kV69eTJ8+HYBXXnmFZ599lvj4eADi4uKIi4ur876JiIjI+adORaqfnx+33HILgYGB
+APTt29d1OlYa36pVq856jp+uST0Vm83mKlB/KTMzkzZt2hAQEODadvHFF5OZmVmndfPy8igqKuLu
+u+922x4UFOT6+scff6Rr1651mk9EREQuDHUqUtu2bUteXp6rSP3piJic/yIjI8nLy6O4uBh/f38A
+9uzZQ1RUVJ3Gt27dGpvNxurVq4mIiDhpn6ioKHbt2kX37t1POY+Pj88JR5NFRETk/FWna1L79u3L
+559/TkVFBYZh8PXXX9OxY8fGziYtQFBQELfccgvjxo2juroah8PB448/ftqjsr9kMpm49957uf/+
++3E4HMDxm6G2bt3q6nPfffcxceJE0tPTAdi3bx8vvvii2zxdunRh+/btZGVlAZCbm9sQuyciIiIt
+lBlg8eLFHDhwAKfTidPpZOrUqdjtdpKSkgDo0KEDRUVFzJ07l+rqaqKiohg4cGCzBpem89Zbb/HQ
+Qw8RExOD2Wzm97//fZ2vRwWYMmUKzz77LJdffjkmk4mAgAAmTZpEjx49ABg1ahTHjh1j2LBhlJaW
+0rZtWyZMmOA2R3R0NM888wxXX301VquViIgIVqxYgdlcp5MBIiIico4xGYZhnM0Ev36U0K/NsttP
+2z6+luVrm/+nU9Dn6/hzybFjxwgICGD79u1ER0c3dxwRERE5h+m1qHLW9u7dC8Dnn3+On59fo7xx
+SkRERC4sOlcqZ+XAgQPcddddHDx4EF9fX+bPn4+Xl1dzxxIREZFznIpUOSvt2rXjiy++aO4YIiIi
+cp7R6X4RERERaXFUpIqIiIhIi6MiVURERERaHBWp56BtS95g05vP1nucYdQ0Qpq6+/LLL9m8eXOt
+/aZMmcKDDz5Y7/lrapp3/0RERKThmAFKS0vZtGkT6enp2O12Ro4c6epQWlrK9OnT3QZVV1djs9kY
+O3Zs06aVM1aw77+kvv0SQx+f02wZ0tLSCAwM5LLLLmvwubdt28Zf//pXPvnkkwafW0RERJqeGcDD
+w4OwsDCcTid5eXluHfz8/HjkkUfcti1cuJBu3bo1XUo5axVFhc22dmVlJY8++igpKSlUV1ezZs0a
+pk6dSqtWrRpsjV//3oqIiMi5zQxgtVqJi4s7aZH6azt27MBsNtOlS5cmCShQWeLgmzlTOLz9W/xD
+2mELaYe3LcDVXrg/g+8/eJOjGTtxlhYR3qM/l/9xEp5e3lQ48lkzeQwVRQU4S4tZct91APiHtGfo
+5DcAqKmuZtviOWR+uRIMg+DOPemTNB4vq59rjWPHjtG7d2/i4+OZN29evfK/9dZbbNq0iR9//BGL
+xcLcuXMpLy93Fan5+fncf//9fP7558TExBATE0NQUJBr/I4dO3j22WfZvHkzBQUFXH/99cyePRsf
+Hx9ycnIYPHgwubm5FBYWut50FRsby6effurK/vTTT5OSkoJhGPTv359XX331vHrbl4iIyPmmXtek
+GobBunXruPrqqxsrj5zExtlP4OFp5rbX1zBwwnTK8nPc2osPZRF15XXc/MoShs36mMIDGfx3zWIA
+fAKCuPmVJfQdM5Hgzj1ImL2KhNmrXAUqQNqi2RzZ+R03vriQW6cvx+JrY8u709zWqKioICMjg507
+d57RPphMJgzDwGw2c8899xAWFuZqGzNmDF5eXmRlZbFs2TKys7Pdxu7evZvf/e53bNu2jT179rBz
+505ef/11AIKDg9m2bRszZsygf//+ZGZmkpmZ6SpQAZ544gk2bNjA5s2b2bVrFwEBAUycOPGM9kNE
+RESaRr2K1IyMDPz9/Wnbtm1j5ZFfcZYUkbXpM/qOegRPLws+9laEXdrPrU/7PtfSvtc1VDsrcRzI
+wB4aSe6P39d5jfSVC+iZOBYvH18wmeiWMIb9qevc+thsNvbu3cv69evrvQ+jR48mLi6OqKgoJk6c
+iMPhcLUVFBSwdOlSpk2bhre3N23btmXIkCFu42+55RZuuukmKioq2LlzJx06dODbb7+t8/rTp0/n
+mWeewWazYTKZmDhxIsuWLav3foiIiEjTqdcbp3bv3u06nSpNozgnG5+AVlhs9lP2KcvP4du3nqOq
+vIzWF3fB5OHJsbKSOs1fWVSAs7yUr2b83W27t/+J6/3yFHx9WCwW5syZw5///Geef/55OnXqxOrV
+q7n00kvJzMykbdu2p70+NTs7m+TkZEpKSujVqxdms9mt0D2dvLw8ioqKuPvuuxtkX0RERKRp1KtI
+zcrKYvDgwY2VRU7CGhBEZbGD6qpKPL28T9pnw8vj6HxDIpH9jh+BzFi3jKxNn7n18fSyUFl84s1T
+3v6BePn4MuSx1/BrE3raLA6HA6vVisViOaN96dKlC++88w4PP/wwr7/+OrNmzSI4OJj8/HwqKirw
+8fE56bgRI0aQnJzMbbfdBsA777zD0qVL3fr4+Phw9OjRE8a2bt0am83G6tWriYiIOKPcIiIi0vTq
+dbq/oKBAN5s0Md/WIbSK6si29+eAYVB8OIvMDSvd+pTkHsLkcfxHWXRoH7vWvH/CPIHtYynM2k1p
+3iEAKooKjjeYTHQaejvfzJmC839HXysc+eRnpruNLy0tJSoqigEDBtR7H5KTk3nttdc4ePAge/bs
+4dtvv6Vjx44AtGvXju7du/P0009jGAa7d+9mwYIFbuP37duHp6cnAD/++KPretRf6tKlC9u3bycr
+KwuA3Nzc/+2eiXvvvZf777/fdfQ1JyeHrVu31ns/REREpOmYDMMwFi9ezIEDB3A6nTidTmw2G3a7
+naSkJFfH6upqnnrqKf72t7/h5/fzXd/FxcWnXWCW/dSnqQHGG8Zp22ubv7ai+VwfD8cLz69nPk7x
+kf0ERcUR3KUnZUdz6Dt6AgD7U9exbckcjlWUExjRgXY9ryJr01quHT/VbZ4d/36b9I9T8LT44Ncm
+lEGPzsTD05Oa6mN8v+RNMr/8GEwmLL42ut92D+0u+/kGuaqqKnr37s2ll17KO++8U2vmX8rIyOCF
+F15g5cqVBAYGkpycTFJSklvhOWrUKDIyMoiPj+eqq64iOzvb9XzeZcuWMWXKFEpLS7nkkkv4v//7
+Pz788EM+/PBDt3VeeuklZsyYgdVqJSIighUrVmA2m6mqquLZZ58lJSUFk8lEQEAAkyZN4oYbbqjX
+foiIiEjTMRlGLVViLVSkNn6Rer6YOXMmgYGBJCYmNncUERERaeHqdU2qyNkIDQ3FZrM1dwwRERE5
+B6hIlSYzbNiw5o4gIiIi54h63TglIiIiItIUVKSKiIiISIujIlVEREREWhwVqSIiIiLS4qhIPYfk
+793FolEDydu9/bT9ti15g01vPtvg6xtGzWnba8tXmneINU/cw6JR17Lsr7dz8D9f12t8c/vyyy/Z
+vHnzKdvT0tIIDQ0lNTX1tPNMmTKFBx98sKHjUVNz+p9PbfmysrIYMmQIF110EfHx8axevbpe40VE
+RBqSGY6/TWjTpk2kp6djt9sZOXKkW6fq6mpWrlxJZmYmhmEQFxfH0KFDMZlMzRL6QuXbqi2R/Qbj
+1zqkydcu2PdfUt9+iaGPzzlln9ryfffOywSERzFo4nQw4NeP6G3O/auLtLQ0AgMDueyyy07aHhoa
+SkJCAu3atWviZLBt2zb++te/8sknn5yyT235/va3v9GpUyeWL1+OYRgn/Hyac/9EROTCYwbw8PAg
+LCwMp9NJXl7eCZ1SU1MpKSnhgQceoLq6mgULFrBjxw4uueSSJg98IfMJCKLv6InNsnZFUWGtfWrL
+V7Dvv1z5p6fw9PI+o/HNpbKykkcffZSUlBSqq6tZs2YNU6dOpVWrVm79goODmTFjRrNkPNl/t79W
+W75t27Yxb948fHx8zmi8iIhIQzIDWK1W4uLiTlmklpeXExERgaenJ56ensTGxtb6JiVpOGsmj6H4
+yH4ASvMOc/PLSwhsH+tqryxx8M2cKRze/i3+Ie2whbTD2xbgaq+prmbb4jlkfrkSDIPgzj3pkzQe
+L+vx19t+NG4E/e59jO0fvsWh77/FFhzONX95Af+L2lPhyGfN5DFUFBXgLC1myX3XAeAf0p6hk9+o
+U77v/vky+1PXUXxkP+tefAgPs1e9xteWvyw/h8+eS2bwpNmkznuBg2kbaRXRwTU/wLFjx+jduzfx
+8fHMmzevXp//W2+9xaZNm/jxxx+xWCzMnTuX8vJyV5E6ePBgMjIyANi/fz9paWl07drVNT4/P5/7
+77+fzz//nJiYGGJiYggKCnLL9vTTT5OSkoJhGPTv359XX33V9Tay3r178/rrr/Pcc8/x2WefER0d
+zXvvvUdsbCw5OTkMHjyY3NxcCgsLiY6OBiA2NpZPP/20Tvn+9re/sXz5cjIyMkhISMBisdRrfG35
+s7OzueWWW/j444956KGH+OSTT+jWrZtrfhERkZMyfiEtLc2YP3++8Wt5eXnGq6++amzdutUoKSkx
+5s6daxQWFhqGYRhFRUWn/fMcnPZPbWqb/3wf/2sLk641CrJ2u237/IWHjA1TJxjHnJVGuSPf+Pjv
+dxvfvPGMq33LgunGqseSDGd5qWHU1Bjfzn3erX353+4wl5u+4gAAIABJREFUlj883MjL2GlUFBUY
+n055wPhq5uNua+zd+ImxevKYM8r3kyX332AczUyv9/ja8pcePWIsGj3IWPPEPcber9cYVeVlRunR
+I25zFBcXG/7+/kafPn1q3YdfmzlzpnHVVVcZJSUltfa96KKLjO3bt7ttGzZsmDFy5EijoqLCyMnJ
+Ma655hrjT3/6k6t90qRJxrXXXmsUFxcbNTU1xp///Ge39l69ehk9e/Y0Nm/ebOTl5Rk33HCDMWrU
+KLc1Fi9ebAwePPiM8v3k4osvNv7zn//Ue3xt+Q8cOGCEh4cbQ4YMMd5//32jpKTEOHDgQK1ZRUTk
+wlanG6cCAgIIDQ1ly5YtvPzyy4SHhxMQEFD7QGl0zpIisjZ9Rt9Rj+DpZcHH3oqwS/u59UlfuYCe
+iWPx8vEFk4luCWPYn7rOrU+PO5NpHdMZb/9Aoq74DY7szCbci9OrS/6y/By6JYwmst8QzD5WfIOC
+3dptNht79+5l/fr19V5/9OjRxMXFERUVxcSJE3E4HHUeW1BQwNKlS5k2bRre3t60bduWIUOGuPWZ
+Pn06zzzzDDabDZPJxMSJE1m2bJlbnylTptCzZ09at27N8OHDSU9Pr/d+NJa65M/OzmbixIncdttt
++Pn5ER4e3kxpRUTkXFGn16K+++679O3bl7i4OPLz8/noo4/YuHEj/fr1q32wNKrinGx8AlphsdlP
+2l5ZVICzvJSvZvzdbbu3v3t/D/PPvwrWwNZUVzkbPuwZqGt+s48vF3Xtfdq5fnmKvT4sFgtz5szh
+z3/+M88//zydOnVi9erVXHrppbWOzczMpG3btidcv/qTvLw8ioqKuPvuu0+b1cvLy/V1SEgIlZWV
+Z7AnDa+u+W02GwMGDGjCZCIicq6rtUgtLy/nyJEjxMXFAcf/5zNkyBCWL1+uIrUFsAYEUVnsoLqq
+8qQ3JHn7B+Ll48uQx17Dr03oGa/j6WWhsrj2m6caWkPlB3A4HFitViwWyxmN79KlC++88w4PP/ww
+r7/+OrNmzap1THBwMPn5+VRUVJz0hqTWrVtjs9lYvXo1ERERZ5QLwMfHh6NHj57x+DPVUPlFRER+
+rdbT/T4+PlgsFnbt2oVhGNTU1LB7926d7m8hfFuH0CqqI9venwOGQfHhLDI3rPy5g8lEp6G3882c
+KTjLSgCocOSTn1m/08WB7WMpzNpNad6h43MUFTTYPpxWA+UvLS0lKirqjI7mJScn89prr3Hw4EH2
+7NnDt99+S8eOHes0tl27dnTv3p2nn34awzDYvXs3CxYscLWbTCbuvfde7r//ftdlBDk5OWzdurVe
+Gbt06cL27dvJysoCIDc3t17jz1RD5RcREfk1M8DixYs5cOAATqcTp9PJ1KlTsdvtJCUlYTKZGDFi
+BGvWrGHNmjUYhkFYWBg33HBDc2eX/7n6oef5eubjvH/PYIKi4ogZcCNlR3Nc7T0Sk/l+yZusfCQR
+TCYsvja633YPQdFxdV7DFhxOz8RkVk36A54WH/zahDLo0Zl4eHo2xi65aYj8FouFyMhIOnToUO/1
+x44dywsvvMCUKVMIDAwkOTmZpKSkOo9PSUlh1KhRtGvXjvj4eO666y6ys7Nd7VOmTOHZZ5/l8ssv
+x2QyERAQwKRJk+jRo0ed14iOjuaZZ57h6quvxmq1EhERwYoVKzCb63RFz1lpiPwiIiK/ZjKMXz2x
+u55qexTVLPvJr5X8yfhalq9t/p8ec3O+jpeWY+bMmQQGBpKYmNjcUURERM57jX+YReQ8ERoais1m
+a+4YIiIiFwQVqSJ1NGzYsOaOICIicsGo03NSRURERESako6kNjJdcyoiIiJSfzqSKiIiIiItjopU
+EREREWlxVKSKiIiISItjhuNv49m0aRPp6enY7XZGjhzp1qmoqIiPPvqI3NxcrFYrv/nNb4iMjGyW
+wCIiIiJy/vMA8PDwICwsjJiYmJN2+uCDD+jQoQNjx45l2LBhLFmypNaH1IuIiIiInCkPAKvVSlxc
+HGFhYSd0qKio4NChQ/Tq1QuANm3a0LNnT7777rumTSoiIiIiF4w6XZNaVVWF0+l0fR8cHExubm6j
+hRIRERGRC1utRaqPjw+hoaFs2rSJyspKMjIyWLt2LSUlJU2RT0REREQuQHU6knr77bdz9OhRFixY
+wJ49e7jqqquw2+2NnU1ERERELlB1euNUYGAgt956q+v71atXExIS0mihREREROTCVqcjqXv37qWy
+shKAPXv2sH37dteNVCIiIiIiDc0MsHjxYg4cOIDT6cTpdDJ16lTsdjtJSUkAHDlyhBUrVuB0OgkK
+CuKuu+7CarU2a3AREREROX+ZDMMwzmaC2p6XOquWa1fH17J8bfP7+/u36PEiIiIiUn96LaqIiIiI
+tDgqUkVERESkxVGRKiIiIiItjopUEREREWlxVKSKiIiISIujIlVEREREWhwVqSIiIiLS4pgBsrOz
+Wbt2Lbm5uXh7ezNo0CA6d+7s6lRTU8Mnn3zCrl278PT05PLLL+eyyy5rttAiIiIicn4zG4bBhg0b
+GDRoEGFhYezZs4eUlBTGjh3relD9119/TVFREX/605+orKzk7bffJigoiOjo6GaOLyIiIiLnIw+T
+ycSIESMIDw/HZDIRGxtLSEgIR44ccXXasmULAwYMwMPDA6vVyhVXXMGWLVuaMbaIiIiInM9OuCa1
+pqaGgoIC2rRp4/q+qKiINm3a8NVXX5Genk5wcDD5+flNHlZERERELgzmX2/YuHEjMTExBAYGAnDs
+2DE8PDwwmUzs3bsXp9NJmzZtqKysbPKwIiIiInJhcCtSMzMz2bx5M0lJSa5tFosFOF6sJiYmApCV
+leW6XlVEREREpKG5TvcfPHiQpUuX8rvf/Q6bzebWKTg4mOzsbNf3Bw4cIDg4uOlSioiIiMgFxQNg
+//79LFy4kOHDhxMSEnJCp169erF+/Xqqq6spKSkhNTWVnj17NnlYEREREbkwmKuqqpg/fz4mk4n3
+3nuP6upqAMLCwhg5ciQA8fHxFBYWMnv2bDw8PBg8ePBJi1kRERERkYZgMgzDOJsJiouLT9s+y24/
+bfv4Wpavbf7aro1t7vEiIiIiUn96LaqIiIiItDgqUkVERESkxVGRKiIiIiItzgkP8xd3uuZURERE
+pOnpSKqIiIiItDgqUkVERESkxVGR2gQ+GjeCQ99vqrWfYdQ0QRoRERGRls8MkJ2dzdq1a8nNzcXb
+25tBgwbRuXNnV6fS0lI2bdpEeno6drvd9ZB/aTgF+/5L6tsvMfTxOc0dRURERKTZmQ3DYMOGDQwa
+NIiwsDD27NlDSkoKY8eOdd005OHhQVhYGE6nk7y8vGaOfH6qKCps7ggiIiIiLYbZZDIxYsQI14bY
+2FhCQkI4cuSIq0i1Wq3ExcVdsEVqWX4OX8+ejGN/Bh5eFlrHdKHHnQ/iH9IOgHcSunPH2xvw9g8E
+YGvKDI5VlNH77nGuOY7u+YH/LJxF0cF9tO3YnSvvfwJveysqHPmsmTyGiqICnKXFLLnvOgD8Q9oz
+dPIbrvU/ey6ZwZNmkzrvBQ6mbaRVRAdXe011NdsWzyHzy5VgGAR37kmfpPF4Wf3q1A5w7Ngxevfu
+TXx8PPPmzWv8D1VERETkNE54BFVNTQ0FBQW0adOmOfK0SGmLXsMWHM7giTMB2J+6zq3Aq4tDaRsZ
+8NeX8PYPZMMr4/lu/lSufOAJfAKCuPmVJez75lN2rV50ytP95QV5fDH1EToOuY1+f3wMZ9nPr2tN
+WzSbnPSt3PjiQry8raS+/SJb3p1G39ET6tQOUFFRQUZGBhaLpb4fj4iIiEiDO+HGqY0bNxITE0Ng
+YGBz5GmRfFuHcGTnZg7v3ExNTTXt+1yLj71Vvea45NYkrK3a4mH2IvbaW8je8kW9xpfl59AtYTSR
+/YZg9rHiGxTsaktfuYCeiWPx8vEFk4luCWPYn7quzu0ANpuNvXv3sn79+nrlEhEREWkMbkdSMzMz
+2bx5M0lJSc2Vp0XqnjAab1sAW/41FcfBvbTvdQ09E8e6FYr1Edg+lsoSR73GmH18uahr7xO2VxYV
+4Cwv5asZf3fb7u1vr1P7LwUFBdUrk4iIiEhjcRWpBw8eZOnSpdx5553YbLbmzNTimDw8ibv+DuKu
+v4PKEgeb3niGja8/xaAJ0wHwMHtRUVTguia15ljVaecrPrzfdT3rTzy9LFQW1//mKW//QLx8fBny
+2Gv4tQmtd/svORwOrFarTvmLiIhIs/MA2L9/PwsXLmT48OGEhIQ0d6YWZ8uCaRTuzwDA289OQLsY
+MAxXuz0skox1y6muquTAd+vZs+GjE+bYt/ETqqsqqSorIW3Ra1w88Ldu7YHtYynM2k1p3iEAKooK
+6hbOZKLT0Nv5Zs4UnGUlx8c68snPTK9b+/+UlpYSFRXFgAED6rauiIiISCMyV1VVMX/+fEwmE++9
+9x7V1dUAhIWFuZ6HunjxYg4cOIDT6cTpdDJ16lTsdvsFc1lA24u7kTrvBUpyD2HU1GAPi+DyMZNc
+7X3uHsfXsyeTsW4Zkf2G0DNx7AlFoC2kHSvGJ1JZXEB0//+jy013ubcHh9MzMZlVk/6Ap8UHvzah
+DHp0Jh6enrXm65GYzPdL3mTlI4lgMmHxtdH9tnsIio6rUzuAxWIhMjKSDh06nM1HJSIiItIgTIbx
+i0OCZ6C4uPi07bPsJ177+Evja1m+tvl/ekxWY40XERERkaan16KKiIiISIujIlVEREREWhwVqSIi
+IiLS4pzwxilxp2taRURERJqejqSKiIiISIujIlVEREREWhwVqSIiIiLS4pgBsrOzWbt2Lbm5uXh7
+ezNo0CA6d+7s6lRbu4iIiIhIQzIbhsGGDRsYNGgQYWFh7Nmzh5SUFMaOHYu/vz+1tYuIiIiINDSz
+yWRixIgRrg2xsbGEhIRw5MgR/P39qa1dRERERKShnXBNak1NDQUFBbRp0+akA2prFxERERE5WycU
+qRs3biQmJobAwMCTDqitXURERETkbLkVqZmZmWzevJnrrrvupJ1raxcRERERaQiuIvXgwYMsXbqU
+3/3ud9hsthM61tYuIiIiItJQzAD79+9n8eLFDB8+nJCQkBM61dYuIiIiItKQzFVVVcyfPx+TycR7
+771HdXU1AGFhYYwcOZLa2kVEREREGprJMAzjbCYoLi4+bfssu/207eNrWb62+Wt7DFZzjxcRERGR
++tNrUUVERESkxVGRKiIiIiItjopUEREREWlxVKSKiIiISIujIlVEREREWhwVqSIiIiLS4qhIlXOG
+YdQ0dwQRERFpImaA7Oxs1q5dS25uLt7e3gwaNIjOnTu7OmVlZfH5559z9OhRTCYTffv25Yorrmi2
+0HLhKdj3X1Lffomhj89p7igiIiLSBMyGYbBhwwYGDRpEWFgYe/bsISUlhbFjx7oeVJ+ZmcmgQYNo
+164deXl5vP7664SFhREVFdW86eWCUVFU2NwRREREpAmZTSYTI0aMcG2IjY0lJCSEI0eOuIrUa665
+xtXepk0b2rdvT3l5eZOHPRcd3vEdO5e9w7HKckpyD9Hn7nFseus57GGRDPn7awDUVFezbfEcMr9c
+CYZBcOee9Ekaj5fVD4DC/Rl8/8GbHM3YibO0iPAe/bn8j5Pw9PIGoCw/h69nT8axPwMPLwutY7rQ
+484H8Q9pB8A7Cd254+0NePsHArA1ZQbHKsroffc41/jPnktm8KTZpM57gYNpG2kV0YGhk9+oNV9D
+7N9H40bQ797H2P7hWxz6/ltsweFc85cX8L+oPRWOfNZMHkNFUQHO0mKW3HcdAP4h7V35AI4dO0bv
+3r2Jj49n3rx5jfozFRERkcZ3wjWpNTU1FBQU0KZNG7fthmFQUlJCamoq5eXldOjQoclCnusOpm2k
+z6gJtOnQje8/nMuNzy8gb/d2SvMOA5C2aDZHdn7HjS8u5Nbpy7H42tjy7jTX+OJDWURdeR03v7KE
+YbM+pvBABv9ds9jVnrboNWzB4STMXsWt05YRfeV1rgKwrsoL8vhi6iNE9BlIwqyP6Z885Rfznz7f
+2e4fwMbZk7nk1lHcOn0Z1sDWfP/BWwD4BARx8ytL6DtmIsGde5AwexUJs1e5FagAFRUVZGRksHPn
+znrtt4iIiLRMJxSpGzduJCYmhsDAQLft6enpzJ49m88//5ybb74Zs9ncZCHPdQHtoglsH4s9NJLw
+nv3xtrfCr00oxUcOAJC+cgE9E8fi5eMLJhPdEsawP3Wda3z7PtfSvtc1VDsrcRzIwB4aSe6P37va
+fVuHcGTnZg7v3ExNTTXt+1yLj71VvTKW5efQLWE0kf2GYPax4hsU7GqrLd/Z7h9AjzuTaR3TGW//
+QKKu+A2O7Mx65bfZbOzdu5f169fXa5yIiIi0TG6VZmZmJps3byYpKemEjp07d6Zz587k5+ezePFi
+rrjiCi655JImC3o+MJlO/LqyqABneSlfzfi7W19vf7vr67L8HL596zmqystofXEXTB6eHCsrcbV3
+TxiNty2ALf+aiuPgXtr3uoaeiWPdCs3amH18uahr7xO21yXf2e4fgMcv/tFjDWxNdZWzztl/EhQU
+VO8xIiIi0jK5KoODBw+ydOlS7rzzTmw22ykHBAUFER8fzw8//KAitQF4+wfi5ePLkMdew69N6En7
+bHh5HJ1vSCSy3xAAMtYtI2vTZ652k4cncdffQdz1d1BZ4mDTG8+w8fWnGDRhOgAeZi8qigpc16TW
+HKtq0HyNOf4nnl4WKotPf/OUw+HAarVisVjOeB0RERFpGTwA9u/fz8KFCxk+fDghISFuHcrLy3n/
+/fc5evQoAAUFBezYsYPw8PCmT3s+MpnoNPR2vpkzBef/jo5WOPLJz0x3dSnJPYTJ4/iVGUWH9rFr
+zftuU2xZMI3C/RkAePvZCWgXA4bhareHRZKxbjnVVZUc+G49ezZ81KD5GnX8/wS2j6UwazeleYeO
+z1FU4NZeWlpKVFQUAwYMqNe8IiIi0jKZq6qqmD9/PiaTiffee4/q6moAwsLCGDlyJFarlU6dOvHv
+f/+bwsJCDMMgPj6eyy+/vJmjnz96JCbz/ZI3WflIIphMWHxtdL/tHoKi4wDoO3oC25bMYWvKDAIj
+OtBp6HCyNq11jW97cTdS571ASe4hjJoa7GERXD5mkqu9z93j+Hr2ZDLWLSOy3xB6Jo6tV5FYW77G
+Hg9gCw6nZ2Iyqyb9AU+LD35tQhn06Ew8PD0BsFgsREZG6oY+ERGR84TJMH5xyO0MFBcXn7Z9lv3E
+axd/aXwty9c2/0+PyWqp40VERESk/vRaVBERERFpcVSkioiIiEiLoyJVRERERFocPZG/kemaVhER
+EZH605FUEREREWlxVKSKiIiISIujIvUcYhg1zR2hUaWlpREaGkpqamqzrF9Tc35/viIiIucSD4Ds
+7Gz++c9/8o9//IMZM2bwww8/nHJASkoKc+bMabKAclzBvv/yyZP3NneMMxIdHU1ISAhhYWH069eP
+f//73yftFxoaSkJCAu3atWvihLBt2zZ+85vfNPm6IiIicnJmwzDYsGEDgwYNIiwsjD179pCSksLY
+sWNPuKknLS2Nqqq6v/ddGk5F0enfW9/SrVy5kp49e/LVV18xYsQIysrKGDFihFuf4OBgZsyY0Sz5
+8vLymmVdEREROTkPk8nEiBEjCA8Px2QyERsbS0hICEeOHHHrWFRUxBdffMGVV17ZTFEvTBWOfJY9
+lMAXUx8h54etLLnvOpbcdx1rJo8BoHB/Bov/ONTtUgBnaTELkwZQXVUJwEfjRrBnwwo+fvT/sTBp
+AJ89+yCVRQWu/jXV1fxn4Ww+fPAmPvzTjXw18zGqykvdchw7dowePXpw9913n/G+mEwm+vfvzz/+
+8Q8ee+wx1/bBgwcTHR1NdHQ0ZrOZHTt2uI3Lzs6mV69e5ObmMnLkSEJCQhg8eLBbtsmTJ9OpUyc6
+duxIUlLSCU9VSElJoUePHoSHh3PZZZexbNkyAHJycujevTuJiYl8+eWXrhy/nN/hcDB69GgiIiKI
+jY3l6aefdr0+uLZ8O3bsICIiwu1SgsLCQkJCQqioqDjjz1JEROR8d8I1qTU1NRQUFNCmTRu37cuX
+L2fgwIF4e3s3WTgBn4Agbn5lCX3HTCS4cw8SZq8iYfYqhk5+A4DA9rHYQtqRvfUr15h9mz6lfa8B
+eHr9/LPKWLeMAQ+/xO1zPsHD7MV3819xtaUtms2Rnd9x44sLuXX6ciy+Nra8O80tR0VFBRkZGezc
+ufOs9+m6665j9+7dFBUVAfDpp5+SmZlJZmYmbdu2PemYw4cPk5iYyG9/+1v27NnDO++842p74okn
+2LBhA5s3b2bXrl0EBAQwceJEV/uiRYuYMGEC8+bNIzs7m3fffZeysjLg+NHbbdu2MWPGDPr37+/K
+8emnn7rGjxkzBpPJREZGBqmpqaxYsYJXX321Tvm6du1KTEwMq1atcvX94IMPuOmmm/Dx8TnLT1JE
+ROT8dUKRunHjRmJiYggMDHRt27p1K15eXnTp0qVJw0ndxF13Bz9+ssT1feaGlcRcc6Nbn0tuTcLa
+qi0eZi9ir72F7C1futrSVy6gZ+JYvHx8wWSiW8IY9qeucxtvs9nYu3cv69evP+u8drsdHx8fDh8+
+XOcx2dnZTJw4kdtuuw0/Pz/Cw8NdbdOnT+eZZ57BZrNhMpmYOHGi60gpwCuvvMKzzz5LfHw8AHFx
+cdxxxx11WrewsJAPPviAl156CS8vL4KCgnjyySd544036pzvgQcecOu/YMEC7rrrrjrvu4iIyIXI
+7WH+mZmZbN68maSkJNc2h8PBhg0bGDVqVJOHk7qJ6DuQ7975B+UFuWAyUXzkABd1ueyU/QPbx1JZ
+4gCgsqgAZ3kpX834u1sfb3/7CeOCgoIaJK/D4aCiooKwsLA6j7HZbAwYMOCE7Xl5eRQVFZ1wGcIv
+s/7444907dr1jLJmZmbSpk0bAgICXNsuvvhiMjMz65QP4Le//S0PP/wwhw4dwmQysWfPHq6++uoz
+yiMiInKhcBWpBw8eZOnSpdx5553YbDZXh127dmEymZg7dy5w/Pq/0tJSpk2bxpgxY5o+8QXK08tC
+ZfHJb57y8DRz8cDfkrFuOWYfK9FXXQ8m0ynnKj6UhS34+JE+b/9AvHx8GfLYa/i1CT1tBofDgdVq
+xWKxnPmOACtWrKBTp05uv2dnqnXr1thsNlavXk1ERMRJ+0RFRbFr1y66d+9+ynl8fHw4evToCdsj
+IyPJy8ujuLjYdSPhnj17iIqKqnNGLy8v7r77bv75z3/i5+fHiBEjMJ3m5yMiIiL/O92/f/9+Fi5c
+yPDhwwkJCXHr0KdPH5KTk11/fuqTnJyM1WptltAXosD2sRRm7aY07xAAFb+48Qmg45DbyFi/nL1f
+ryH2mptOGL/36zVUV1XiLCshbdFrdBh06/EGk4lOQ2/nmzlTcJaVHJ/bkU9+Zrrb+NLSUqKiok55
+tLCuvvrqK8aNG8eTTz55VvP8xGQyce+993L//ffjcBw/OpyTk8PWrVtdfe677z4mTpxIevrxfdq3
+bx8vvvii2zxdunRh+/btZGVlAZCbmwscPyJ7yy23MG7cOKqrq3E4HDz++OP1PrNwzz33MH/+fN5/
+/32d6hcREakDc1VVFfPnz8dkMvHee++57loOCwtj5MiRzRxPfmILDqdnYjKrJv0BT4sPfm1CGfTo
+TDw8PQHwDWpLQLsYSnIOEhAefcJ4s7cPK8bdSWVJIdFX3UCXm34ulHokJvP9kjdZ+UgimExYfG10
+v+0egqLjXH0sFguRkZF06NDhjPLfdNNNmEwm2rdvz8yZM7nlllvOaJ6TmTJlCs8++yyXX345JpOJ
+gIAAJk2aRI8ePQAYNWoUx44dY9iwYZSWltK2bVsmTJjgNkd0dDTPPPMMV199NVarlYiICFasWIHZ
+bOatt97ioYceIiYmBrPZzO9//3v+8pe/1CtjWFgYnTt3Zu/evcTFxdU+QERE5AJnMgzDOJsJfv2o
+n1+bZT/x2sZfGl/L8rXN/+tnuZ5v4+vjm9efJjDiYuKud78p6KNxI7jsrocI7danwdaS+rvvvvu4
+5JJLeOCBB5o7ioiISIun16KeJw7vSOXwjlQ6Dkk4RY+z+reInKV169axbt06XcctIiJSR+bau0hL
+dqyygqXJN+Nl9ePKB57Ew+zV3JHkF8rKyoiLi8NutzN37tyzvulMRETkQqHT/S18vIiIiMiFSKf7
+RURERKTFUZEqIiIiIi2OilQRERERaXFUpF5ADKPmjMZ9NG4Eh77f1MBpTq6qqopx48ZRVlbWJOud
+zJQpU3jwwQfrPa6m5sw+XxGRU0lPT+fll1+uU9+z/fuzOf/+TUtLIzQ0lNTU1NP2O9O/n2tT29/f
+teXLyspiyJAhXHTRRcTHx7N69ep6jW+pavv9q8/v55nwAMjOzuaf//wn//jHP5gxYwY//PCDW6e0
+tDSeeuopnnvuOdefHTt2NFooaXgF+/7LJ0/e29wxajVy5Ehat/7/7N15WFXV+sDx74HDfBhEhEAQ
+kJAhNTRnLU3Q7FpWYpph9+bYdNMms8jKSrK0W94ccEjNBtHUMjVzSFEyyQFNnPAKgiCozKOHQdi/
+P/x58sRwDohA+n6ep+ehvda79ruXh3UWe6+9d2usra112y5dusTo0aPx9PTEw8ODJUuW6MU89thj
+ODk54eXlhaenJ4MHD+b48eNNmnd8fDwPPPDATWs/JycHZ2dnxo4dy9ixY/n+++91ZZWVlbz66qv4
++voSGBhYrX8MOX/+PA8//DA+Pj5069aNX3/9Va98zpw5uv22b9++1nb27dvHwIED+de//qWX3/WG
+DRtG9+7d65Wfofj//Oc/hIWFMWjQIDIyMhrU7t+5fw8ePEhISAju7u4EBATwww8/1Cs/Q/HSv83b
+v3feeSexsbF89tlnBvdV0/hZH805/rq6uhIaGooaG5cXAAAgAElEQVS7u3uDcr8RxozfhvKbOnUq
+fn5+pKSkEBsby7333luv+JbK0OfPULmhz49BVVVVyqpVq5Tz588rVVVVSmJiovLBBx8ohYWFyjW/
+//67smXLFqUmhYWFdf73EdT5nyGG2r/V4xtLRvx+ZduMiQ2K3TT1CSUj/vdGzqi6qKgoZdiwYdW2
+Hz58WFm3bp1SVVWlHDlyRDEzM1PS09N15Y8++qjyxRdfKIqiKFVVVcr8+fOVjh071rqf5cuXK6tW
+raq1fObMmcq///3veuW+c+dOJSQkpF4x1+8vOjq6zjrZ2dlKr169aiz76KOPlMcff1ypqKhQcnJy
+lE6dOim7du0yev/9+/dXFixYoCiKoiQkJCgeHh56/Xu9O++8s9Z2/vnPfypr166ttXzlypVKSEiI
+0q1bN6Nzq0/81KlTlXnz5jWo7b9r/1ZWVirDhg1TDhw4oFRVVSnbt29XrKysam3/r+oTL/3bfP1b
+UlKi+Pv7K6dOnap1X7WNn8ZqKeOvIQ0Znw25kfH7Gn9/fyU2NraRMjKeMd8fN8rQ56+uckOfH0NM
+VCoVo0ePpm3btqhUKnx8fHBxceHSpUu6iaxWq8XGxqZ+s1/RKC6eOMSuWS+yfcYEvn9hKOcP7WH9
+cw+y44M/z4rmpyXx63/fZMPkR/hu/P38Nv9tKivKACgtyGXjy6H8OvcNMk8dYf1zQ1j/3BC2z9B/
+qHzy3p/Z9NrjrJ0YzOapo0g7uFuvvOJyMXv+8xqrn76Pza+Ppuhiml75lStX6NKlC2PHjm3wsf7n
+P/9h1qxZ1bZ36dKF0NBQVCoVHTp0wNbWttbnjapUKkJDQ0lISKh2+ebChQs89NBD7Ny5kyFDhui2
+5+bm8sQTT+Di4kLv3r05efKkXtyJEycYM2YMAQEB3HHHHYwdO5bS0lIAMjMz6dy5M2FhYezduxdv
+b2+8vb0JCQnR65sZM2bg5+dHhw4dGDdunN6jyUaOHMnbb7/Nyy+/jFarrXe/ffHFF7z33nuo1Woc
+HR157bXXWLp0qVGxBQUFHDlyhOeeew4APz8/xo0bx+LFi+udh6IoODo61liWnp7Ohx9+yOuvv17v
+do2Nb926dYPaNqQl96+JiQk//vgj3bt3R6VSMWjQIDp27MixY8eMarM+8dK/zde/1tbWTJ06lfnz
+59dap7bx01jNNf6GhIToxk21Wl3tKq2h8dnQ+Nq9e3cOHz7MyJEjcXJyonv37iQlJQHGjd+G8ps6
+dSr+/v4kJiYSGhpa73hD+aenp9OtWzeysrIYM2YMLi4ueu3f6PfHNZ9++imxsbE1lhn6/NVVXp/P
+T02qrUmtqqoiLy8PJycn3bbS0lJSU1OJiori22+/5fDhw0bvQNy4jKOx9Bj/Jk6+nTj2w3Ie+ngV
+2YnHKcm+CEDRhVS8+g5h2GfrGb7wZ/LPJ/G/7esAsLR3ZNhn6+k5MRzngC6ERm4lNHIrg2f8+SWQ
+sm8bh7+ZS98XPuDxpTu596WPuFJWqpfDH2si6fjYeB6btxErh9Yc+36ZXnlpaSlJSUnVBhCjjzEj
+g8LCQgIDA2utU1VVxfjx43nxxRf1Pp9/rbN8+XJ69OiBicmfH+/Vq1cTHBzMhAkT+Oabb2jVqpWu
+bOLEiZiZmZGamsrGjRtJT0/XazMxMZFRo0YRHx/P2bNnOXnypO5L0NnZmfj4eObPn0+/fv1ITk4m
+OTmZX375RRf/3nvvERMTQ1xcHKdPn8be3p7w8HBdua+vL3v27MHNzY3evXuzf7/x638rKys5f/48
+fn5+zJkzhw0bNtCxY0cSExONilcUBa1WqzcodurUqUH/jhqNptZBcuLEiURERGBn4LnJtTEmXqvV
+Nrj92vxd+vf6fFNSUvD39693+4bipX+bt38fffRRNmzYUGOZMeNnXZpz/P3ll19042abNm2qtWlo
+fDY0vl5r44033uD06dO4uLjoJuPGjN+G8pszZw4JCQl4eXmxZcuWescbk//FixcJCwvj0Ucf5ezZ
+s6xcuVJXdiPfH4BuiUlJSQlFRUVUVVXpnaS8pq7PnzHlxnx+alLtjVOxsbG0b98eBwcH3ba77roL
+rVaLl5cXOTk5rF27FpVKRZcuXYzekWg4e3dvHDx8sHP1xMHDBwu7Vtg4uVJ06Tw2Tnfg0eN+ACq0
+JRRmpGDn6knWmWMEGNn+yU1f03XMSzh6Xx047dt6Y9/WW6/OPU+9TOv2V1v06vMA/9uxTq9co9GQ
+kpLS4LVQKSkp+Pj41Fnn/fffx87OjhkzZlQrmzZtGh988AGKonDPPfewatUqXdmyZcuIjIwkJiam
+2i9HXl4eGzZsIDs7GwsLC9q0acOgQYO4ePGirs4jjzwCXH0xw+nTp/H19eXAgQNGH9u8efPYunUr
+Go0GgPDwcLp168a8efN0dUxMTJg6dSr/+Mc/GDhwIDt27KBz584G29ZqtZiZmWFiYsLu3bspKioi
+ICBA70t7yJAh5OTk6MX179+fTz75BAcHB4KCgvj888+ZMmUKsbGxhIeH4+zsbPTxVVVVkZ6ezv79
++5kyZUq18uXLl2NtbU1oaGi9B9D6xPv6+vLzzz/z0EMP6Z0Rq+v4Dfk79O/1Pv30UwYOHIinp6fR
+7RsbL/3bPP17jaOjI5cvX6a8vLzamShjxs+6NNf4a4gx47Mx42tERARdu3YFrp55rPe6yJvImPzT
+09P55ptvGDBgAEC1K9sN/f4AWLFiBZs3b6a0tJSdO3fy/vvv89JLLzFixAi9enV9/owpr+vzUxe9
+SWpycjJxcXGMGzdOr5KHh4fuZ1dXV/r27UtCQoJMUpuYSlXzz5dzMzmw7CMqtJdpfWcgKhNTrlwu
+NrrdwgupOHjUPUCZqP/8qFg5tKayorxandou9RqjvLwcM7O6X+l64sQJPvjggxrLPv74Y8aPH19j
+2aBBg/j222+ZPn06n3zyiW4wAHR/3V7/l/1fpaenM3nyZIqLi+nWrRtqtZqCggIjjgqys7MpLCys
+tgyipr5KTU1lypQpDBs2zOgvHI1Gg6IolJWV8dNPPwHw22+/4erqqquzdevWOttYu3Yt77zzDkOH
+DqVXr168+eab1e5MrcuGDRt49913uffee7nzzjv1ytLS0pg5c2atl5EMqU/80KFDWb58OUOHDmXZ
+smW6s0KGjr8uLb1/rxcdHc2SJUuq3ThkLEPx0r/N07/XU6vVNU4CjBk/69Jc468hhsZnY8fX64/N
+xcWFsrIyo3O4mYzNX6PR6CaotWnI9wfAW2+9xdNPP03Xrl2pqKjg6NGjmJqa1li3ts+fMeV1fX7q
+ojsfn5GRwYYNGxg1apTBD5FKpdI7lS+aV8ynr+Pd70EGvbOIrk9OxrVTj2p1TM3MKSvKrzFe4+xG
+QXrKDedRUFBAeXn1yasx3N3dSUtLq7PO2rVrG3SZrV27duzcuZPAwEB69uxJdHS0rszZ2Znc3Fzd
+GtOajB49mtGjR7Nt2zYiIiK4//77q9WxtLSsdrYHrq4z02g0bNu2jVOnTun+O3LkiF69ZcuW8cAD
+D/Dqq6+ydOnSeq0B79ixo96Z3djYWDp16mR0vKenJytXriQmJobZs2dz7NixesUPHz6cY8eOkZaW
+Vu0M88aNGzE1NaVfv374+voyYsQIjh07hq+vL3l5eQbbrk/8V199xcCBA4mNjW3wZc+atOT+vebQ
+oUM8/fTTrF+/njvuuMPotusTL/3bvP1bWlpKVVVVjd/PxoyfdWmu8dcQQ+OzseOrIbWN3zdbY+Vv
+zPdHXd/P06dPZ+nSpTzyyCMsW7asxjp1ff6MKW/o58cErp6tWLNmDSNHjsTFxUWvQklJCevWrdN9
+IeTn5/Pbb78REGDsxWRxsxVnXUD1/380FF44x+nta6vVcfDwIT81kZLsCwCUFv75Be/3wEgOf/s5
+BenJ/99eBsd//LJeOZSUlODl5WXwr73a+Pj4UFhYqHcZ53q5ubl4enqyaNGiBrWvUqmYPHkyGzZs
+4J133tGtnXF3d6dz587MnDkTRVFITEzUu1QFcO7cOd1flmfOnKnxpozAwECOHz9OamoqAFlZWbr9
+Pvvsszz//PO6s6+ZmZl6g9Dbb7/N7t27iY2N5cEHH6z3sT3zzDO8//77lJeXc+nSJSIjI5kwYYLR
+8bt376awsBC4un5qzZo1uhtR6sPNzY38fP0/hF544QXOnDmj+2/dunV06tSJM2fO6J0dURSFvn37
+VrvEZGw8XB2b3Nzc6p23IS25f+HqpC40NJS1a9fWeomvtv41Nh6kf5u7f7dv3653w8z1DI2fhjTX
++GuIofHZmPHVGLWN3zdbY+RvzPdHXd/PRUVFBAYGMmzYMN5//30uXLhQYxt1ff4Mld/I50ddUVHB
+119/jUqlYvXq1VRWVgJXf2HHjBmDjY0Nd955J99//z1FRUWYmJjQs2dPo9c7iJuv54Q3iV+/hCNR
+83Fo54vf4JGk7t+pV0fj3JauYZPZOv1pTM0tsXFyJfitBZiYmuIbPBylspLo2S9zpVSLpX0rOg03
+/ksCwNzcHE9PT3x9fRt0DCqVikmTJjFnzhz+85//VCtXFKVB7f7VtUXm1w/GUVFRjB8/Hnd3d4KC
+gnjqqaf0FufPmzePiIgI3n77bTp27Mizzz5b7VmH3t7efPjhh9x3331YWVnRrl07fvrpJ9RqNRER
+EcyaNYtevXqhUqmwt7dn+vTpuuUyzzzzzA09O+/pp58mJSWFzp07Y2ZmxqxZs+p1Jik+Pp4XXniB
+4uJifHx82Lp1a53LH2pTUlKCpaVlveOuMTU15YcffiAnJ6dBd5JbWlpSXGz8MhdjteT+vXz5MoMH
+D8bU1JThw4frLmPec8891S6T19S/9YmX/m2+/lUUhTlz5tR6udTQ+GlIc46/hhganw2Nr8aoa/y+
+2W40f2O+P+r6fra1tWXq1KnA1c/wu+++W62Ooc+fMeUNpVJu8NN3/QL3miw0cDfoNAO7N9S+ra3t
+LR1/O6moqKBv37588MEHN/XB+H9HOTk5PPTQQw1e29lYfH19OXPmTI1lY8eOZciQIYwaNapBbZeW
+lnLHHXdw6dIlLCws6h3/2muv4e3tzQsvvFDvWOlfw6R/63Yz+3fmzJmcO3euzkdz3ej4KeOvqI2h
+z58xn8+Guvl/JghhJDMzMzZt2sTTTz/Nvffe2+AnBdyqkpKSmDjx6vNthw4dyqOPPtok+/300091
+b6G7dqWlJpMmTSI8PJxt27YxbNiweuc3a9YsRowYUe8v+Llz53L48GEyMjJ49dVX6xV7Penfmkn/
+Gudm9e/x48c5ceIEX331VZ3t1DR+/v7777VeAs7Ozta7QUbGX1ETQ58/Yz+fDSVnUlt4vBC3g61b
+t/Lbb78xY8aMWu8sFQ0n/XtzSf8KcXPIJLWFxwshhBBC3I7kOVJCCCGEEKLFkUmqEEIIIYRocWSS
+KoQQQgghWhyZpAohhBBCiBZHDVffTb5z506ysrKwsLAgODi42hulEhMT2blzJ0VFRdjb2zNw4MB6
+vR9WCCGEEEIIY6kVRSEmJobg4GDc3Nw4e/YsUVFRTJkyRXfneUZGBj/99BNPPPEELi4u5OTk6N6c
+IYQQQgghRGNTq1QqRo8erdvg4+ODi4sLly5d0k1SY2JiGDhwIC4uLgANemWhEEIIIYQQxqr2xqmq
+qiry8vJwcnLSbbt06RJ9+/blp59+IisrCw8PD+69917Mzc2bNFkhhBBCCHF7qHbjVGxsLO3bt8fB
+wUG3raioiF27dtG1a1dGjRpFTk4OO3fubNJEhRBCCCHE7UNvkpqcnExcXBxDhgzRq2RjY8MjjzyC
+q6srVlZW9OzZk8TExCZNVAghhBBC3D50k9SMjAw2bNjAqFGj0Gg0epXatGlDdna27v//Wi6EEEII
+IURjMgFIS0tjzZo1jBw5Undz1PV69uxJdHQ0paWlKIrCvn376NChQ5MnK4QQQgghbg/qiooKvv76
+a1QqFatXr6ayshIANzc3xowZA4Cvry+FhYUsX76cyspKvLy8GDhwYHPmLYQQQgghbmEqRVGUG2mg
+qKiozvKFdnZ1lk8zsHtD7V97TNatGi+EEEIIcTuS16IKIYQQQogWRyapQgghhBCixZFJqhBCCCGE
+aHFkkiqEEEIIIVocmaQKIYQQQogWRyapQgghhBCixZFJqmh08euXsv+LWc2dhhBCCCH+xtQA6enp
+7Ny5k6ysLCwsLAgODiYgIACAkpIS5s2bpxdUWVmJRqNhypQpTZ+xEEIIIYS45akVRSEmJobg4GDc
+3Nw4e/YsUVFRTJkyBVtbW2xsbHjjjTf0gtasWUOnTp2aKWUhhBBCCHGrU6tUKkaPHq3b4OPjg4uL
+C5cuXarxbUgnTpxArVYTGBjYlHne1i7nZrLro8mETI/k4IrZZByNpVU7XwbPWApAVWUl8euWkLx3
+CygKzgFd6TFuGmZWNro2kvf+zPENyyktyMXKwYm7Rz6HR/cBAJRfLubQl5+QcXQfJqZq7hz4KJ2G
+j0dlYmrU/suKC/h9SQQXjx/A1sUdjYs7Fhp7vfz3Rc6gIC0JEzNzWrcPpMuTL2Lr4q6rc+XKFbp3
+705QUBArVqy42V0qhBBCiBZO/dcNVVVV5OXl4eTkVK2yoijs3r2bkSNHNkly4k/avGx+nfsGHQaN
+oPcz71B++c/XrR79LpLMhCM8NGcNZhZWHPxyDoe//ZyeE94EIGXfNg5/M5f7p/0XR29/CtKTyU0+
+rYuPjZyBmbWG4Qu3UKEtYWfEC5haWHLXw/80av+xke9ham7JiMXbqdCWsPuTV/UmqUe/W4TGuS0h
+4QsASDu4W28CDVBaWkpSUhLm5uaN23FCCCGE+FuqduNUbGws7du3x8HBoVrlpKQkbG1tadOmTZMk
+J/50OTeTTqET8Ow9CLWlFdaOzrqyhC2r6Bo2BTNLa1Cp6BQ6kbSDu3XlJzd9TdcxL+Ho7Q+AfVtv
+vPsNAaC8pIhzv/9Ct3+9hompGguNPUFPvMCZHeuN2n95cSGp+3fRc/wbmJqZY2nXCre7e+vFWrd2
+4dLJOC6ejKOqqhKPHvdjaddKr45GoyElJYU9e/Y0Wp8JIYQQ4u9L70xqcnIycXFxjBs3rsbKiYmJ
+eHt7N0liQp/a0po77upebXtZYR7l2hJ+m/+23nYLWzvdz4UXUnHw8Kmx3eLMdCztWmFurdFts3Nt
+R3FmulH7L8pMx9K+FeYau2pl13QOnYCFxp7D38ylICMFj2796Ro2RW+iDeDo6FhrG0IIIYS4vegm
+qRkZGWzYsIEnn3wSjUZTY+XU1FRCQkKaLDlhmIWtA2aW1gx6ZxE2Tq411tE4u1GQnkIrzw7Vymza
+uFJamEeFtkR3Cb7o0nls2rgZtX8re0fKigqorCjD1MyixjoqE1P8H3wC/wefoKy4gP1LPyR28QcE
+v6n/1IiCggKsrKzkkr8QQgghrl7uT0tLY82aNYwcORIXF5daK+fl5dV4M5VoRioVfoMf5/clEZRf
+LgagtCCX3OQEXRW/B0Zy+NvPKUhPBqA4K4PjP34JgIXGnnY9BhL31WcoVZWUXy7mj9UL8Q0ZbtTu
+rVu70MqrA/Frl4CiUHQxleSYLXp1Dq/6nPy0pKv7s7HD3r09KIpenZKSEry8vBgwYEBDekEIIYQQ
+txh1RUUFX3/9NSqVitWrV1NZWQmAm5sbY8aM0VWsrKxEq9VibW3dXLmKWnQJm8yx9V+w5Y0wUKkw
+t9bQecQk3RpU3+DhKJWVRM9+mSulWiztW9Fp+ARdfJ/n3+Pgitmsf+4fmJia4jPgYe56+Cmj93/f
+yx+zb8G7rJ0UgqOXP+0HPMTlnExdeZs7O3FwxWyKsy6gVFVh59aOXhOn67Vhbm6Op6cnvr6+N9gb
+QgghhLgVqBTlL6e06qmoqKjO8oV2ta9VBJhmYPeG2jd0ZvfvHi+EEEIIcTuS16IKIYQQQogWRyap
+QgghhBCixZFJqhBCCCGEaHGqvXFKtCyyplUIIYQQtyM5kyqEEEIIIVocmaQKIYQQQogWRyapQhhJ
+UaqaOwUhhBDitqEGSE9PZ+fOnWRlZWFhYUFwcDABAQG6SpWVlWzZsoXk5GQURcHf35/BgwejUqma
+LXFhnLKifFY/fR+9Jr2F3wOjANj7+Vtknj7K8AWbmzm7v4+8c//j4JefMPjdJc2dihBCCHFbMFEU
+hZiYGIKDg3nllVd48MEHWb9+vd4NOwcPHqS4uJgXXniB5557jgsXLnDixIlmTFvUh4WtAyn7tgNQ
+daWCrDPHmjmjv5/SwvzmTkEIIYS4rahVKhWjR4/WbfDx8cHFxYVLly7p7hzXarW0a9cOU1NTTE1N
+8fHxMXjXuWg5zG3sKC3IpbQgl+zE49i39SI/7ayuvKqykvh1S0jeuwUUBeeArvQYNw0zKxsA8tOS
+OPb9F+QknaS8pJC2XfrR65npmJpZAHA5N5N9kTMoSEvCxMyc1u0D6fLki9i6uAOwMrQzT3wZg4Wt
+AwBHouZzpfQy3ce+rovf9dFkQqZHcnDFbDKOxtKqnS+DZyw1mN/FE4c4uXElV8q0FGddoMfY19m/
+7CPs3DwZ9PYio45v8+uj6f3sOxz/YRkXjh1A49yW/q/MxvYOD0oLctk+YyKlhXmUlxSx/rkhANi6
+eOjyA7hy5Qrdu3cnKCiIFStW3LR/SyGEEOJ2UW1NalVVFXl5eTg5Oem2de7cmbi4OP744w9KSkpI
+TEwkMDCwSRMVDXel9DKevUJI3b+Tc7E7cAvqq1d+9LtILp08xENz1vDYvE2YW2s4/O3nuvKiC6l4
+9R3CsM/WM3zhz+SfT+J/29ddF78IjXNbQiO38tjnG/HuO0Q3ATSWNi+bX+e+QbseAwld+DP9JkcY
+nV/G0Vh6jH8TJ99OHPthOQ99vIrsxOOUZF80Kh4gNnIGHR8bz2PzNmLl0Jpj3y8DwNLekWGfrafn
+xHCcA7oQGrmV0MitehNUgNLSUpKSkjh58mS9jlsIIYQQNas2SY2NjaV9+/Y4ODjottnb2+Pq6srh
+w4f59NNPadu2Lfb29k2aqGi4yooyvO8bSuqBXeSmnMbZ72698oQtq+gaNgUzS2tQqegUOpG0g7t1
+5R497sejW38qy8soOJ+Enaun3pIB69YuXDoZx8WTcVRVVeLR434s7VrVK8fLuZl0Cp2AZ+9BqC2t
+sHZ0Njo/e3dvHDx8sHP1pG3XfljYtcLGyZWiS+eNigfo8uRkWrcPwMLWAa8+D1CQnlyv/DUaDSkp
+KezZs6decUIIIYSomd7D/JOTk4mLi2PcuHF6lb799lt69uyJv78/ubm5bN68mdjYWHr37t2kyYqG
+s3fzorQwj7Zd+sF1N7yVFeZRri3ht/lv69W3sLXT/Xw5N5MDyz6iQnuZ1ncGojIx5crlYl1559AJ
+WGjsOfzNXAoyUvDo1p+uYVP0JpqGqC2tueOu7tW2G5PfNdffx3ftZ2PjTdR//ipYObSmsqLc6Nyv
+cXR0rHeMEEIIIWqm+2bOyMhgw4YNPPnkk2g0Gl0FrVbLpUuX8Pf3B65+EQ8aNIhNmzbJJPVvpv8r
+szHX2Osug8PVm6rMLK0Z9M4ibJxca4yL+fR1AoaG4dl7EABJuzeSun+XrlxlYor/g0/g/+ATlBUX
+sH/ph8Qu/oDgN+cBYKI2o7QwT7cmtepKhdE5G5PfzYy/xtTMnLKium+eKigowMrKCnNz8wbvRwgh
+hBBXmQCkpaWxZs0aRo4ciYuLi14FS0tLzM3NOX36NIqiUFVVRWJiolzu/xuyvaMdFpq//LupVPgN
+fpzfl0RQ/v9nR0sLcslNTtBVKc66gMrk6sqQwgvnOL19rV4Th1d9Tn5aEgAWNnbYu7cHRdGV27l5
+krR7E5UVZZw/tIezMfV49JUR+d3U+P/n4OFDfmoiJdkXrrZRmKdXXlJSgpeXFwMGDKhXu0IIIYSo
+mbqiooKvv/4alUrF6tWrqaysBMDNzY0xY8Zw7e7/7du3s337dhRFwc3NjaFDhzZz6qKxdAmbzLH1
+X7DljTBQqTC31tB5xCQcva+ePe854U3i1y/hSNR8HNr54jd4JKn7d+ri29zZiYMrZlOcdQGlqgo7
+t3b0mjhdV95j7Ovsi5xB0u6NePYeRNewKfWaJBrK72bHA2ic29I1bDJbpz+NqbklNk6uBL+1ABNT
+UwDMzc3x9PTE19fX6DaFEEIIUTuVolx3yqsBDD2KaqFd9bWD15tmYPeG2r/2mCyJF0IIIYS4dchr
+UYUQQgghRIsjk1QhhBBCCNHiyCRVCCGEEEK0OGrDVcTfmaxpFUIIIcTfkZxJFUIIIYQQLY5MUoUQ
+QgghRIsjk1RRb/Hrl7L/i1n1jlOUqpuQjfH27t1LXFycwXoRERG8+OKL9W6/qqp5j08IIYS4lagB
+0tPT2blzJ1lZWVhYWBAcHExAQICuUmFhIZs3byYrKwsrKyseeOABPD09my1p8feTd+5/HPzyEwa/
+u6TZcjh69CgODg7cc889jd52fHw8r776Kjt27Gj0toUQQojbkVpRFGJiYggODsbNzY2zZ88SFRXF
+lClTdDfVfP/999x11108+eSTZGdn89VXXzFx4kS56UYYrbSw7vfe30xlZWW89dZbREVFUVlZyfbt
+25k7dy6tWrVqtH1kZ2c3WltCCCGEAPW1155e4+Pjg4uLC5cuXcLW1pbS0lIuXLjAv/71LwCcnJzo
+2rUrhw4d4v7772+uvEUTKisu4PclEVw8fgBbF3c0Lu5YaOx15flpSRz7/gtykk5SXlJI2y796PXM
+dEzNLCgtyGX7jImUFuZRXlLE+ueGAGDr4sHgGUsBqKqsJH7dEpL3bgFFwTmgKz3GTcPMyka3jytX
+rtC9e3eCgoJYsWJFvfJftmwZ+/fv58yZM5ibm7N8+XK0Wq1ukpqbm8vzzz9PdHQ07du3p3379jg6
+OuriT5w4waxZs4iLiyMvL48HH3yQyMhILKEwn0sAACAASURBVC0tyczMJCQkhKysLPLz8/H29gau
+/h798ssvutxnzpxJVFQUiqLQr18//vvf/8ofeUIIIUQdqq1JraqqIi8vDycnJ922iooKysvLdf/v
+7OxMVlZW02Qoml1s5HuYmKoZsXg7A9+cx+XcTL3yogupePUdwrDP1jN84c/kn0/if9vXAWBp78iw
+z9bTc2I4zgFdCI3cSmjkVt0EFeDod5FcOnmIh+as4bF5mzC31nD428/19lFaWkpSUhInT55s0DGo
+VCoURUGtVjNp0iTc3Nx0ZRMnTsTMzIzU1FQ2btxIenq6XmxiYiKjRo0iPj6es2fPcvLkSRYvXgxc
+/V2Ij49n/vz59OvXj+TkZJKTk3UTVID33nuPmJgY4uLiOH36NPb29oSHhzfoOIQQQojbRbVJamxs
+LO3bt8fBwQEAS0tLXF1d2b9/P2VlZSQlJbFz506Ki4ubPFnR9MqLC0ndv4ue49/A1MwcS7tWuN3d
+W6+OR4/78ejWn8ryMgrOJ2Hn6knWmWNG7yNhyyq6hk3BzNIaVCo6hU4k7eBuvToajYaUlBT27NlT
+72OYMGEC/v7+eHl5ER4eTkFBga4sLy+PDRs28Pnnn2NhYUGbNm0YNGiQXvwjjzzCww8/TGlpKSdP
+nsTX15cDBw4Yvf958+bx4YcfotFoUKlUhIeHs3HjxnofhxBCCHE70XuYf3JyMnFxcYwbN06v0uOP
+P050dDSrVq3C3d2de++9l8TExCZNVDSPosx0LO1bYa6xq7XO5dxMDiz7iArtZVrfGYjKxJQrl437
+I6asMI9ybQm/zX9bb7uFbfX9XX8Jvj7Mzc1ZsmQJL730Eh9//DF+fn5s27aNu+++m+TkZNq0aVPn
++tT09HQmT55McXEx3bp1Q61W601065KdnU1hYSFjx45tlGMRQgghbhe6SWpGRgYbNmzgySefRKPR
+6FVycHDgscce0/3/tm3bcHFxabosRbOxsnekrKiAyooyTM0saqwT8+nrBAwNw7P31TOQSbs3krp/
+l14dUzNzyoqq3zxlYeuAmaU1g95ZhI2Ta525FBQUYGVlhbm5eYOOJTAwkJUrV/Laa6+xePFiFi5c
+iLOzM7m5uZSWlmJpaVlj3OjRo5k8eTIjRowAYOXKlWzYsEGvjqWlJTk5OdViW7dujUajYdu2bbRr
+165BeQshhBC3IxOAtLQ01qxZw8iRI2ucfKakpFBWVgbA2bNnOX78ON26dWvaTEWzsG7tQiuvDsSv
+XQKKQtHFVJJjtujVKc66gMrk6sqRwgvnOL19bbV2HDx8yE9NpCT7AgClhXlXC1Qq/AY/zu9LIij/
+/7OvpQW55CYn6MWXlJTg5eXFgAED6n0MkydPZtGiRWRkZHD27FkOHDhAhw4dAHB3d6dz587MnDkT
+RVFITExk1apVevHnzp3D1NQUgDNnzujWo14vMDCQ48ePk5qaCqBbs61SqXj22Wd5/vnndWdfMzMz
+OXLkSL2PQwghhLidqMrLy5U5c+agUqkwNzensrISADc3N8aMGQPA/v37OXToEOXl5Tg6OvLggw/i
+7OwMGH43/EK72i8TA0xTlDrLb/Td8xJ/Y/FwdeK5b8G7FF1Kw9HLH+fArlzOyaTnhDcBSDu4m/j1
+S7hSqsWhnS/uXe8ldf9O7p82V6+dEz9+ScLPUZiaW2Lj5ErwWwswMTWlqvIKx9Z/QfLen0Glwtxa
+Q+cRk3C/5z5dbEVFBd27d+fuu+9m5cqVBnO+XlJSErNnz2bLli04ODgwefJkxo0bpzfxHD9+PElJ
+SQQFBXHvvfeSnp7OvHnzANi4cSMRERGUlJTQsWNH/vGPf/DDDz/www8/6O3nk08+Yf78+VhZWdGu
+XTt++ukn1Go1FRUVzJo1i6ioKFQqFfb29kyfPp2hQ4fW6ziEEEKI24lKUQzMEg2QSeqtHX8rWbBg
+AQ4ODoSFhTV3KkIIIYQwQG24ihC3BldX12rrrYUQQgjRMskkVdw2hg8f3twpCCGEEMJI1Z6TKoQQ
+QgghRHOTSaoQQgghhGhxZJIqhBBCCCFaHJmkCiGEEEKIFkcmqcJouSmn+W78QLITj9dZL379UvZ/
+MavR968oVXWWG8qvJPsC29+bxHfj72fjq4+T8ce+esU3t7179xIXF1dr+dGjR3F1deXgwYN1thMR
+EcGLL77Y2OlRVVX3v4+h/FJTUxk0aBB33HEHQUFBbNu2rV7xQgghbi1quPrlEB0dTU5ODiqVip49
+e9KnTx9dpaqqKnbs2MHp06cxNTWlV69e3HPPPc2WtGge1q3a4Nk7BJvWTf9K3Lxz/+Pgl58w+N0l
+tdYxlN+hlZ9i39aL4PB5oMBfHxHcnMdnjKNHj+Lg4FDr756rqyuhoaG4u7s3cWYQHx/Pq6++yo4d
+O2qtYyi/qVOn4ufnx6ZNm1AUpdq/T3MenxBCiKanBkhOTiY4OBh3d3eys7NZvHgxbm5ueHl5AbBv
+3z4KCwv597//TVlZGV9++SWOjo54e3s3Z+6iiVnaO9JzQniz7Lu0MN9gHUP55Z37H33//QGmZhYN
+im8uZWVlvPXWW0RFRVFZWcn27duZO3curVq10qvn7OzM/PnzmyXH7Oxsg3UM5RcfH8+KFSuwtLRs
+ULwQQohbixqgf//+ug1OTk54eHig1Wp12w4fPszo0aMxMTHBysqKPn36cPjwYZmk3ia2z5hI0aU0
+AEqyLzLs0/U4ePjoysuKC/h9SQQXjx/A1sUdjYs7Fhp7XXlVZSXx65aQvHcLKArOAV3pMW4aZlY2
+AGx+fTS9n32H4z8s48KxA2ic29L/ldnY3uFBaUEu22dMpLQwj/KSItY/NwQAWxcPBs9YalR+h776
+lLSDuym6lMbuOS9jojarV7yh/C/nZrLro8mETI/k4IrZZByNpVU7X137AFeuXKF79+4EBQWxYsWK
+evX/smXL2L9/P2fOnMHc3Jzly5ej1Wp1k9SQkBCSkpIASEtL4+jRo9x11126+NzcXJ5//nmio6Np
+37497du3x9HRUS+3mTNnEhUVhaIo9OvXj//+97+6t5F1796dxYsX89FHH7Fr1y68vb1ZvXo1Pj4+
+ZGZmEhISQlZWFvn5+boxwcfHh19++cWo/KZOncqmTZtISkoiNDQUc3PzesUbyj89PZ1HHnmEn3/+
+mZdffpkdO3bQqVMnXftCCCFaJt2aVEVRKC4u5uDBg2i1Wnx9fYGrl/oLCwtxcnLit99+IyEhAWdn
+Z3Jzc5stadG0Bs9YSmjkVkIjt2Jp51itPDbyPUxM1YxYvJ2Bb87jcm6mXvnR7yK5dPIQD81Zw2Pz
+NmFureHwt5//pY0ZdHxsPI/N24iVQ2uOfb8MuHp2c9hn6+k5MRzngC66PK6fABrKr9s/X+GxeRvR
+tGlL8FsL6x1vTP7avGx+nfsG7XoMJHThz/SbHKFXXlpaSlJSEidPnqytm+ukUqlQFAW1Ws2kSZNw
+c3PTlf3yyy8kJyeTnJxMmzZtqsVOnDgRMzMzUlNT2bhxI+np6Xrl7733HjExMcTFxXH69Gns7e0J
+Dw+v1sYbb7zB6dOncXFxYdasq2uOnZ2diY+PZ/78+fTr10+Xx/UTQEP5zZkzh4SEBLy8vNiyZUu9
+443J/+LFi4SFhfHoo49y9uxZVq5cWVd3CyGEaAF0k9SEhAQiIyOJjo5m2LBhqNVXX0Z15coVTExM
+UKlUpKSkcOHCBczMzCgrK2u2pEXLUV5cSOr+XfQc/wamZuZY2rXC7e7eenUStqyia9gUzCytQaWi
+U+hE0g7u1qvT5cnJtG4fgIWtA159HqAgPbkJj6JuxuR/OTeTTqET8Ow9CLWlFdaOznrlGo2GlJQU
+9uzZU+/9T5gwAX9/f7y8vAgPD6egoMDo2Ly8PDZs2MDnn3+OhYUFbdq0YdCgQXp15s2bx4cffohG
+o0GlUhEeHs7GjRv16kRERNC1a1dat27NyJEjSUhIqPdx3CzG5J+enk54eDgjRozAxsaGtm3bNlO2
+QgghjKV7LWpAQAABAQHk5uaybt06+vTpQ8eOHTE3NweuTlbDwsKAqzdaXbuUJm5vRZnpWNq3wlxj
+V2N5WWEe5doSfpv/tt52C1v9+ibqP9/Qa+XQmsqK8sZPtgGMzV9tac0dd3Wvs63rL7HXh7m5OUuW
+LOGll17i448/xs/Pj23btnH33XcbjL129vGv61evyc7OprCwkLFjx9aZq5mZme5nFxeXFvNHqrH5
+azQaBgwY0ISZCSGEuFHqv25wdHQkKCiIU6dO0bFjR+DqJb309HQ8PT0BOH/+PM7Ozn8NFbchK3tH
+yooKqKwoq/GGJAtbB8wsrRn0ziJsnFwbvB9TM3PKigzfPNXYGit/gIKCAqysrHR/+NVXYGAgK1eu
+5LXXXmPx4sUsXLjQYMy1pTmlpaU13pDUunVrNBoN27Zto127dg3KC8DS0pKcnJwGxzdUY+UvhBCi
+5THRarWsXbtW9wWTl5fHiRMn9C6HdevWjT179lBZWalbt9q1a9fmylm0INatXWjl1YH4tUtAUSi6
+mEpyzJY/K6hU+A1+nN+XRFB+uRiA0oJccpPrd7nYwcOH/NRESrIvXG2jMK/RjqFOjZR/SUkJXl5e
+DTqbN3nyZBYtWkRGRgZnz57lwIEDdOjQwahYd3d3OnfuzMyZM1EUhcTERFatWqUrV6lUPPvsszz/
+/PO6ZQSZmZkcOXKkXjkGBgZy/PhxUlNTAcjKyqpXfEM1Vv5CCCFaHrWVlRV+fn78+OOP5OfnoygK
+QUFB9OrVS1cpKCiI/Px8IiMjMTExISQkBBeXlvksSdH07nv5Y/YteJe1k0Jw9PKn/YCHuJzz581T
+XcImc2z9F2x5IwxUKsytNXQeMQlHb3+j96FxbkvXsMlsnf40puaW2Di5EvzWAkxMTW/GIelpjPzN
+zc3x9PTU3ZBYH1OmTGH27NlERETg4ODA5MmTGTdunNHxUVFRjB8/Hnd3d4KCgnjqqaf0bp6KiIhg
+1qxZ9OrVC5VKhb29PdOnT6dLly5G78Pb25sPP/yQ++67DysrK9q1a8dPP/2kW9t+MzVG/kIIIVoe
+lfLXJ2bXU1FRUZ3lC+1qXqt4zTQDuzfUvqG1sRJ/Y/Gi5ViwYAEODg66teFCCCHErezmn+YQQjQK
+V1dXNBpNc6chhBBCNAmZpArxNzF8+PDmTkEIIYRoMiaGqwghhBBCCNG0ZJIqhBBCCCFaHJmkCiGE
+EEKIFkcmqUIIIYQQosWRSapoMXJTTvPd+IFkJx6/Ke0rStVNafeavXv3EhcXd1P3UZejR4/i6urK
+wYMHb0r7VVU3t/+EEEKI65kApKamsnLlSj799FM+++wz9u3bp1eppKSEXbt2sXDhQr755ptmSVTc
++qxbtcGzdwg2rRv/RRF55/7HjvefbfR2r3f06FESEqq/ieqxxx7DyckJLy8vPD09GTx4MMePN/5E
+3NXVldDQUNzd3Ru97fj4eB544IFGb1cIIYSojRogOTmZ4OBg3N3dyc7OZvHixbi5ueHl5QWAiYkJ
+bm5ulJeXk52d3Zz5iluYpb0jPSeE35S2Swvzb0q7AGVlZbz11ltERUVRWVnJ9u3bmTt3Lq1atdLV
++fjjjxk/fjyKorBw4UJGjx7NsWPHGjUPZ2dn5s+f36htXiO/90IIIZqaCUD//v11Z1+cnJzw8PBA
+q9XqKllZWeHv74+bm1vzZCma1ebXR3M25id+futfrBk3gF2zXqSsME9Xfjk3k82vP0FpYR6//vdN
+1owbwPYZE3Xl5ZeL2bdwBuueGcz3z/+D+HVLUKoqdeXbZ0xk/XNDWP/cEL56PIj8tCS9/VdVVvLH
+mkh+ePFhfvj3Q/y24B0qtCV6dZL3/sym1x5n7cRgNk8dRdrB3QCUFuSy8eVQfp37Bpmnjuj2c31+
+AFeuXKFLly6MHTu23v2zbNky9u/fz5kzZzh//jx9+/bV+/25nkqlIjQ0lISEBN3l8/T0dLp160ZW
+VhZjxozBxcWFkJAQXUxBQQETJkygXbt2+Pj4MHPmTCor/+y/kJAQvL298fb2Rq1Wc+LEiWrHNmPG
+DPz8/OjQoQPjxo2r9iayqKgounTpQtu2bbnnnnvYuHEjAJmZmXTu3JmwsDD27t2r28/1+QkhhBA3
+g+5h/oqiUFJSwqlTp9BqtQ16x7i4dSXt3siA1z7BwtaBmM+mcejrz+j7wvu6cm1eNr/OfYMOg0bQ
++5l3KL/85yQoNnIGZtYahi/cQoW2hJ0RL2BqYcldD/8TgMEzlurqfjd+YLV9H/0uksyEIzw0Zw1m
+FlYc/HIOh7/9nJ4T3gQgZd82Dn8zl/un/RdHb38K0pPJTT4NXD07O+yz9Zz7/RdOb/uOwe8uqfH4
+SktLSUpKwtzcvEH9o1KpUBQFtVrNpEmTaq1XVVXF8uXL6dGjByYmfy4Jv3jxImFhYUyaNInFixeT
+n//nmd+JEydib29PUlISRUVFDB06FGtra1555RUAfvnlF11dV1fXavt87733+O2334iLi8PGxoZX
+XnmF8PBw5s2bB8B3333Hm2++yYYNGwgKCiIhIYE//vgDuHp2Nj4+nvXr17No0SJ27NjRoP4RQggh
+6kv3LZmQkEBkZCTR0dEMGzYMtVpeRiX+1PGxcVi1aoOJ2gyf+x8h/fBevfLLuZl0Cp2AZ+9BqC2t
+sHZ0BqC8pIhzv/9Ct3+9hompGguNPUFPvMCZHeuN3nfCllV0DZuCmaU1qFR0Cp2oO1MKcHLT13Qd
+8xKO3v4A2Lf1xrvfkHodn0ajISUlhT179tQrDmDChAn4+/vj5eVFeHg4BQUF1epMmzYNLy8vvL29
+OXToEKtWrdIrT09PJzw8nBEjRmBjY0Pbtm0ByM/P5/vvv+eTTz7BzMwMR0dH3n//fZYuXVptH7WZ
+N28eH374IRqNBpVKRXh4uO5MKcBnn33GrFmzCAoKAsDf358nnnii3v0ghBBCNCbdTDQgIICAgABy
+c3NZt24dffr0oWPHjs2Zm2ihHDx8KCvWn4ipLa25467u1eoWZ6ZjadcKc+s/3zlv59qO4sx0o/ZV
+VphHubaE3+a/rbfdwtZO93PhhVQcPHzqcwg1cnR0bFCcubk5S5Ys4aWXXuLjjz/Gz8+Pbdu2cffd
+d+vqXFuTWhuNRsOAAQOqbU9OTsbJyQl7e3vdtjvvvJPk5GSjcsvOzqawsLDaMobrj/XMmTPcdddd
+RrUnhBBCNJVqp0sdHR0JCgri1KlTMkkVNSq6kIrGua1RdW3auFJamEeFtgQzK5ur8ZfOY9PGuPXN
+FrYOmFlaM+idRdg4Vb+UDaBxdqMgPYVWnh1qbcfUzJyyorpvniooKMDKyqrBl/wDAwNZuXIlr732
+GosXL2bhwoUNaud6np6eZGdnU1RUhK2tLQBnz57V3dRoSOvWrdFoNGzbto127drVWMfLy4vTp0/T
+uXPnWtuxtLQkJyen3vkLIYQQDWWi1WpZu3at7gsoLy+PEydO6C43CgGQsm87lRVllF8u5uh3i/AN
+fsyoOAuNPe16DCTuq89Qqiopv1zMH6sX4hsy3Lgdq1T4DX6c35dEUH65GLh6M1Ru8p+PevJ7YCSH
+v/2cgvSrZxeLszI4/uOXes04ePiQn5pISfaFq21cd+MXXH3MmpeXV41nMw2ZPHkyixYtIiMjg7Nn
+z3LgwAE6dKh9wlwfjo6OPPLII7z++utUVlZSUFDAu+++W+dZ2eupVCqeffZZnn/+ed0yhMzMTI4c
+OaKr89xzzxEeHq57fNa5c+eYM2eOXjuBgYEcP36c1NRUALKyshrj8IQQQohaqa2srPDz8+PHH38k
+Pz8fRVEICgqiV69eukrr1q3j/PnzlJeXU15ezty5c7Gzs2PcuHHNmLpoSmoLS356/UnKivPxvnco
+gQ8/ZXRsn+ff4+CK2ax/7h+YmJriM+Bh7qpHfJewyRxb/wVb3ggDlQpzaw2dR0zSrUH1DR6OUllJ
+9OyXuVKqxdK+FZ2GT9BrQ+Pclq5hk9k6/WlMzS2xcXIl+K0FmJiaAlcv2Xt6ejbohsEpU6Ywe/Zs
+IiIicHBwYPLkyY36u7Fs2TJefvll2rdvj1qt5p///KfupiljREREMGvWLHr16oVKpcLe3p7p06fT
+pUsXAMaPH8+VK1cYPnw4JSUltGnThjfffFOvDW9vbz788EPuu+8+rKysaNeuHT/99JOsXRdCCHHT
+qBRFUW6kgb8+yuavFtrZ1Vk+zcDuDbV/7RKoxN+ceLj6CKp7nnoZ1049DNa9EVWVlUQ91ZtH5v5g
+9HKClmTBggU4ODgQFhbWLPu/cuUK9vb2HD9+HG9v72bJQQghhGgs8lpUYaQb+lumTsWZGQBcPH4A
+tYUV1jfhjVNNwdXVlTZt2jT5flNSUgCIjo7GxsbmprxxSgghhGhqcq1ONKvLOZf49fM30eZmYWph
+yb1TZmFi+vf8WA4fbuQ620Z0/vx5nnrqKTIyMrC2tubrr7/GzMysyfMQQgghGptc7pf4OsuFEEII
+IZrD3/OU1d/IjU4CmzteCCGEEKI5yJpUIYQQQgjR4sgkVQghhBBCtDgySRVCCCGEEC2OGiA1NZXo
+6GhycnJQqVT07NmTPn366Cqlp6ezc+dOsrKysLCwIDg4mICAgGZLWgghhBBC3NrUAMnJyQQHB+Pu
+7k52djaLFy/Gzc0NLy8vFEUhJiaG4OBg3NzcOHv2LFFRUUyZMkVuyhFCCCGEEDeFGqB///66DU5O
+Tnh4eKDVaoGr7/4ePXq0rtzHxwcXFxcuXbokk1QhhBBCCHFT6B5BpSgKJSUlnDp1Cq1WW+s7zKuq
+qsjLy8PJyanJkhRCCCGEELcX3SQ1ISGBzZs3oygKTz31FGp1zY9QjY2NpX379jg4ODRZkkIIIYQQ
+4vaim4kGBAQQEBBAbm4u69ato0+fPnTs2FGvcnJyMnFxcYwbN67JExVCCCGEELePao+gcnR0JCgo
+iFOnTultz8jIYMOGDYwaNQqNRtNkCQohhBBCiNuPiVarZe3ateTk5ACQl5fHiRMnaNu2ra5SWloa
+a9asYeTIkbi4uDRXrkIIIYQQ4jahtrKyws/Pjx9//JH8/HwURSEoKIhevXoBUFFRwddff41KpWL1
+6tVUVlYC4ObmxpgxY5ozdyGEEEIIcYtSKYqi3EgDRUVFdZYvtLOrs3yagd0bat/QY7CaO14IIYQQ
+QtSfvBZVCCGEEEK0ODJJFUIIIYQQLY5MUoUQQgghRIsjk1QhhBBCCNHiyCRVCCGEEEK0ODJJFUII
+IYQQLY5MUoUQQgghRIujBkhNTSU6OpqcnBxUKhU9e/akT58+ukqGyoUQQgghhGhMaoDk5GSCg4Nx
+d3cnOzubxYsX4+bmhpeXF8aUCyGEEEII0ZjUAP3799dtcHJywsPDA61Wq9tmqFwIIYQQQojGpL72
+g6IolJSUcOrUKbRaLb6+vnoVDZULIYQQQgjRWHST1ISEBDZv3oyiKDz11FOo1Wq9iobKhRBCCCGE
+aCy6mWZAQAABAQHk5uaybt06+vTpQ8eOHTG2XAghhBBCiMZS7RFUjo6OBAUFcerUqRoDDJULIYQQ
+Qghxo0y0Wi1r164lJycHgLy8PE6cOEHbtm0BMFQuhBBCCCFEY1NbWVnh5+fHjz/+SH5+PoqiEBQU
+RK9evQAwVC6EEEIIIURjUymKotxIA0VFRXWWL7Szq7N8moHdG2rf1ta2RccLIYQQQoj6k9eiCiGE
+EEKIFkcmqUIIIYQQosWRSaoQQgghhGhx5In8N5msaRVCCCGEqD85kyqEEEIIIVocmaQKIYQQQogW
+Ryapt4HNr4/mwrH9BuspSlUTZCOEEEIIYZgaIDU1lejoaHJyclCpVPTs2ZM+ffrUGBAVFUVRURGT
+Jk1q0kTFzZV37n8c/PITBr+7pLlTEUIIIYS4OklNTk4mODgYd3d3srOzWbx4MW5ubnh5eelVPnr0
+KBUVFc2Rp7jJSgvzmzsFIYQQQggdNUD//v11G5ycnPDw8ECr1epVLCws5Ndff+XBBx9k586dTZvl
+be5ybib7ImdQkJaEiZk5rdsH0uXJF7F1cQdgZWhnnvgyBgtbBwCORM3nSulluo99XddGztlT/LFm
+IYUZ52jToTN9n38PC7tWlBbksn3GREoL8ygvKWL9c0MAsHXxYPCMpbr97/poMiHTIzm4YjYZR2Np
+1c5XV15VWUn8uiUk790CioJzQFd6jJuGmZWNUeUAV65coXv37gQFBbFixYqb36lCCCGEaNF0j6BS
+FIWSkhJOnTqFVqvF19dXr+KmTZsYOHAgFhYWTZ7k7e7od4vQOLclJHwBAGkHd+tN8Ixx4WgsA179
+BAtbB2I+m8ahr+fS94X3sLR3ZNhn6zn3+y+c3vZdrZf7tXnZ/Dr3DToMGkHvZ96h/PKfj9Y6+l0k
+mQlHeGjOGswsrDj45RwOf/s5PSe8aVQ5QGlpKUlJSZibm9e3e4QQQghxC9LdOJWQkEBkZCTR0dEM
+GzYMtfrPR6geOXIEMzMzAgMDmyXJ2511axcunYzj4sk4qqoq8ehxP5Z2rerVRsfHxmHVqg0majN8
+7n+E9MO/1iv+cm4mnUIn4Nl7EGpLK6wdnXVlCVtW0TVsCmaW1qBS0Sl0ImkHdxtdDqDRaEhJSWHP
+nj31yksIIYQQtybdTDQgIICAgAByc3NZt24dffr0oWPHjhQUFBATE8P48eObM8/bWufQCVho7Dn8
+zVwKMlLw6NafrmFT9CaK9fF/7N17XFRl/sDxz8AwMDBcRIHlDpoieAlNwduqeUt/pW5qlpm7pWJl
+G9Zuq6VWtkUXa81VEy+luW1SqeWtFNJMNE1JDRWFFEFUTOQ2A8MM1/n94Xa2WZCbCKTf9+s1r5c8
+z/d5nu+Z7bV8Oec557j5d6C0WN+gMWoHR37XpXe19lJDAWUmI98te9Gq3d7ZpV79v+bu7t6gnIQQ
+Qghx66r2xil3d3fCw8M5ffo0Xbt2QavyWQAAIABJREFUJS0tDZVKxZo1a4BreweNRiNLliwhKiqq
+2RO+HalsbOk86iE6j3qI0mI9h1a/zsGVrzL0haUA2KjtMBsKlD2pVRW139xW9PMFZT/rL2ztNJQW
+NfzmKXtnN+wcHBn+0gqc2nk3uP/X9Ho9Wq1WLvkLIYQQAhuTycSGDRvIy8sDoKCggJSUFHx9fQGI
+iIggOjpa+UycOBEvLy+io6PRarUtmftt4+j6JRReSAfA3skFV7/2YLEo/S4+gaR/u43K8lIu/rCX
+c4nbq81x/uDXVJaXUl5STPJnK7hjyB+s+t38O1CYdRZj7mUAzIaC+iWnUhEy4gG+XxVDWUnxtbH6
+fPIzUuvX/x9Go5GgoCAGDx5cv3WFEEIIcUtTa7VaQkJC2LJlC4WFhVgsFsLDw+nTp09L5yb+w+OO
+biStXUjx1ctYqqpw8QmgT9R8pT/isdkciF1A+rdbCew7nJ6TZ1UrAnVefnw5ZzKlRQUED/g/wkZP
+se739KXn5Gh2zn8UW40DTu28GTrvPWxsbevMr8fkaE5sep+vnp8MKhUaRx3dJ8zAPbhzvfoBNBoN
+gYGB1W7YE0IIIcTtSWWx/OqUXCMUFRXV2r/cpfrew1+bU8fydc3v7Ox8S48XQgghhLgdyWtRhRBC
+CCFEqyNFqhBCCCGEaHWkSBVCCCGEEK1OtUdQidZF9rQKIYQQ4nYkZ1KFEEIIIUSrI0WqEEIIIYRo
+daRIFUIIIYQQrY4aICsriz179pCXl4dKpSIyMpJ+/fopQcnJyWzduhU7OzulbfTo0XTp0qX5MxZC
+CCGEELc8NUBGRgZDhw7Fz8+P3NxcVq5ciY+PD0FBQQCYzWZ69erFqFGjWjJXIYQQQghxm1ADDBo0
+SGlo164d/v7+mEwmpc1kMuHk5NT82QkhhBBCiNuS8ggqi8WC0Wjk9OnTmEwmq3eom81mcnNziYuL
+o6qqitDQUHr27NkiCQshhBBCiFufUqSmpqayfft2LBYLU6ZMQa3+7yNUu3TpgslkIigoiLy8PDZs
+2IBKpaJHjx4tkrQQQgghhLi1KZVoaGgooaGh5Ofns3HjRvr160fXrl0B8Pf3VwZ4e3vTv39/UlNT
+pUgVQgghhBA3RbVHULm7uxMeHs7p06evO0ilUmFjI0+vEkIIIYQQN4eNyWRiw4YN5OXlAVBQUEBK
+Sgq+vr4AGI1GNm7cSEFBAQCFhYV89913hIaGtljSQgghhBDi1qbWarWEhISwZcsWCgsLsVgshIeH
+06dPHwCcnJy44447+PzzzykqKsLGxobIyEi6d+/ewqkLIYQQQohblRqge/futRad4eHhhIeHN1tS
+QgghhBDi9iYbS4UQQgghRKsjRaoQQgghhGh1pEgVQgghhBCtjhSpQgghhBCi1ZEiVQghhBBCtDpS
+pAohhBBCiFZHitTb0PFNqzn0/hsNHmexVN3QuvmZaXw2bQi5Z08qbRWlZjY9OZJNT47kowfv4vKJ
+Qze0hhBCCCFuDWqArKws9uzZQ15eHiqVisjISPr162cVePbsWXbv3k1RURGurq4MGTKEDh06tEjS
+ovkVnP+JpA/fYcTLq6r1Zf94gF0xM9HoXJQ2O3st41fEW8U5tvEgsO8wnNp6KW1qewfGx+4EYPvs
+STcpeyGEEEL81qgBMjIyGDp0KH5+fuTm5rJy5Up8fHwICgoCIDs7my+//JKHHnoILy8v8vLyKC0t
+bcm8RTMzGwpr7W8T2JHR72yoNcbB1Z3I6XObMi0hhBBC3KLUAIMGDVIa2rVrh7+/PyaTSWlLTExk
+yJAheHldOwPWtm3bZk5T3IjSYj3fr4rh55OHcfbyQ+flh73OVekvvJDOic/fJy/9FGVGA749BtDn
+8fnY2tlj1ueTsCAKs6GAMmMRm54cCYCzlz8jFqyu1/oJC6IounIBAGPuz4xZtAk3//qfha+qrOT4
+xlVk7P8KLBY8Q3sSMXUOdlonJaaiooLevXsTHh7O2rVr6z23EEIIIVon9S//sFgsGI1GTp8+jclk
+omPHjkrQlStX6N+/P19++SVXr17F39+f3//+92g0mhZJWjTMwdhXsNU4MGFlAuUmI9++81erIrXo
+chZB/UfS/8+vUlVRQfzLU/kpYSOh907GwdWdMe9u4vz3u0iL/6zGy/11+XUx+9m0IQ0en/xZLDmp
+x7jv7U+xs9eS9OHbHP14CZHTX1BizGYz6enp8t+kEEIIcYtQitTU1FS2b9+OxWJhypQpqNVKF0VF
+RXzzzTeMGDECNzc3tm3bxu7duxk1alSLJC3qr6zYQNahb3jow0Rs7TTY2mnwubMvpoJcJcY/4m4A
+yk1GDNmZuHgHcvXMCUIbsE7B+TN88thA5ef+T72Kf69BtYyov9Sv1jPsxRXYOTgC0G18FNv/9pBV
+karT6cjMzMTR0bFJ1hRCCCFEy1Iq0dDQUEJDQ8nPz2fjxo3069ePrl27AuDk5MTYsWNxc3MDIDIy
+kq1bt7ZMxqJBinIu4eDaxuqmpv9Vkp/D4Q/epNxUQts7wlDZ2FJRUtygdeqzJ7UxSg0FlJmMfLfs
+Rat2e+fqx+Pu7t7k6wshhBCiZaj/t8Hd3Z3w8HBOnz6tFKkeHh7k5uYqRapOp2veLEWjaV3dKS3S
+U1leiq2dfY0xiYtmE3rvZAL7Dgcg/dutZB36xirG1k5DaVHtN0/dKJWNCktlpVWbvbMbdg6ODH9p
+BU7tvGsdr9fr0Wq1cslfCCGEuAXYmEwmNmzYQF5eHgAFBQWkpKTg6+urBEVGRrJnzx7MZjMWi4UD
+Bw7QqVOnlspZNIBjWy/aBHXi+IZVYLFQ9HMWGYlfWcUUX72MyubaI3MNl8+TllD9jKibfwcKs85i
+zL0MgNlQ0OS56jx8uHh0H1gslBbrrzWqVISMeIDvV8VQ9p+zu2Z9PvkZqVZjjUYjQUFBDB48uMnz
+EkIIIUTzU2u1WkJCQtiyZQuFhYVYLBbCw8Pp06ePEtSxY0cMBgNr1qyhsrKSoKAghgxp+A0womUM
+fPYtDrz3MhtmDMM9qDPtB99HSV6O0h85/QWOb1rFsbhluAV0JGTERLIO7baaQ+fpS8/J0eyc/yi2
+Ggec2nkzdN572NjaNlme3SfMYO+i2Wx4fAReYXcx8Jk3AegxOZoTm97nq+cng0qFxlFH9wkzcA/u
+rIzVaDQEBgZa3fAnhBBCiN8ulcVisdzIBEVFRbX2L3e5/l5IgDl1LF/X/M7OzjJeCCGEEOIWI69F
+FUIIIYQQrY4UqUIIIYQQotWRIlUIIYQQQrQ61R5BJW4tsqdVCCGEEL9FciZVCCGEEEK0OlKkCiGE
+EEKIVkeKVFFvFktVS6dwUyUnJ+Pt7U1SUlKTzVlSUkJwcDDBwcHY29uze/fuugc1Uk35N+f6Qggh
+RFNSA2RlZbFnzx7y8vJQqVRERkbSr18/4NqbfJYuXWo1qLKyEp1Ox6xZs5o/Y9EiCs7/RNKH7zDi
+5VUtnUqDmc1m/vKXv/D555+jUqkYOHAg77zzDv7+/lZx3t7ejB8/Hj8/vyZb29HRkYyMDAB69+7d
+qDni4+O59957adOmjdLm5OREZmamVVxN+TfF+kIIIURLUANkZGQwdOhQ/Pz8yM3NZeXKlfj4+BAU
+FISTkxPPP/+81aBPP/2Ubt26tUjComWYDYUtnUKjxcTEcOrUKY4cOYKLiwufffYZSUlJ1YpUT09P
+li1b1kJZ1q5bt24cO3as1pjWnL8QQgjRUDYAgwYNUs6+tGvXDn9/f0wmU40DUlJSUKvVhIWFNV+W
+osWY9flsfXY8+xY/T87pY2x6ciSbnhxJwoIoAAovpLPx8RFWWwHKjEV8OnUwleWlAGyfPYlziV+y
+Y96f+HTqYL5542lKDQVKfFVlJT9+GssXT4/miz/fx3fvvUS5yWiVR0VFBT169OCxxx5r8DGcOnWK
+iIgIfH19cXZ2Ztq0aYwbN07pHzZsmHJJXK1Wk5KSovTt3buXMWPGMHToUO644w62b99O+/btGTly
+pBLTu3dvPv74YwYMGICXlxdjxowhNze33vlVVFSwYMECQkJC6NSpE1OnTq3zqQy/Vlv+zbG+EEII
+cTMoe1ItFgvFxcUkJSVhMplqfAe6xWLh22+/ZeDAgc2apGg5Dq7ujHl3E5FRc/EM7cH42J2Mj93J
+iAWrAXDz74DOy49Lx75Txpw/tAv/XoOxtbNX2tK/3crg597hgVVfY6O244eP3lX6kj+L5cqpH7jv
+7U+5f+k2NI46jn68xCoPs9lMeno6p06davAx/PGPfyQ2NpYXX3yxxuJx165dZGRkkJGRgYeHR7X+
+hIQElixZQmRkJG+++SaHDx/m8OHDXLhwQYlZt24dGzZs4MKFC2g0GmbPnl3v/F555RUSExM5cuQI
+aWlpuLq6Mnfu3HqPryv/m72+EEIIcTMoRWpqaiqxsbHs2bOHMWPGoFZXf4Rqeno6zs7OjfpFKG5d
+nUc+xJmvNyk/ZyR+RftB91nFdL1/Kto2Htio7ehw91guHd2v9KV+tZ6ek2dh5+AIKhXdxkdxIelb
+q/E6nY7MzEz27t3b4PzGjh3L/v37SUpKIjAwkHnz5mE2m+s9PjQ0lC5dutCxY0dGjRpFu3btCAgI
+4Ny5c0rMnDlz8Pb2RqPR8Kc//YkdO3bUe/6lS5fy+uuvo9PpUKlUzJ07l61bt1rFnDhxAg8PD+Wz
+bdu2es/fFOsLIYQQzU2pRENDQwkNDSU/P5+NGzfSr18/unbtahV89uxZgoODmz1J0boFRA7hh3X/
+wFRwFVQqiq5c5Hdhd1033s2/A6XFegBKDQWUmYx8t+xFqxh7Z5dq49zd3Rud45133snOnTs5duwY
+M2bMIC0tjY0bNzZoDpVKVeO//1eXLl3Iz8+v15y5ubkYDIZq2xj+91jrsye1Meq7vhBCCNHcqp0u
+dXd3Jzw8nNOnT1crUrOyshg2bFizJSdaD1s7DaVFNd88ZWOr5o4hfyD9222oHbQE/34U1FLEFV3O
+QufpC4C9sxt2Do4Mf2kFTu28a81Br9ej1WrRaDSNPo4ePXqwcOFCJk6c2Og56nK9P+ZsbGyoqKiw
+amvbti06nY74+HgCAgJuWk6tYX0hhBCiIWxMJhMbNmwgLy8PgIKCAlJSUvD19a0WXFBQIK/RvE25
++XegMOssxtzLAJh/deMTQKfhE0jfu43MAwl0GDS62vjMAwlUlpdSVlJM8mcr6Dj0/msdKhUhIx7g
++1UxlJUUX5tbn09+RqrVeKPRSFBQEIMHD25Q3hUVFfTv35+PPvoIg8HA1atX+fjjj5VHrDWVDRs2
+YDab0ev1vPLKK0ydOrVaTGBgIDt27MBisShnWlUqFU888QQzZ85Er792djknJ+emnDVt6fWFEEKI
+hrDRarWEhISwZcsWFi1axJo1awgICKBPnz5WgZWVlZhMJhwdHVsoVdGSdJ6+9Jwczc75j7I5eiz7
+Fr9AVWWl0u/o7oGrX3sqSs24+lY/i6i2d+DL2Q+zZdZYPDqHEzZ6itLXY3I07e7oylfPT2bzrD/w
+zZvRlOTnWI3XaDQEBgbWeENfbdRqNcuWLePTTz8lJCSELl26YDKZWL16dQO/gdo5OjoSERFBWFgY
+/fr149lnn60WM3/+fBISEggICODpp59W2mNiYoiIiKBPnz6EhYUxduxYsrOzmzS/1rC+EEII0RAq
+i8ViuZEJ6npUzXKX6nsLf21OHcvXNX9dZ3Zl/I2Nb4jvV76GW8AddB71kFX79tmTuGvKs3h3i2iy
+tVqT3r1789ZbbzFkyJCWTkUIIYS4ZchrUUWT+DkliZ9Tkug0fPx1Im7ob6FW7wb/1hNCCCHE/6j+
+nCkhGqCi1Mzm6DHYaZ3o/9TfsVHbtXRKQgghhLgFSJEqboja3oEJKxNqjblvYVwzZdMykpKSWjoF
+IYQQ4pYjl/uFEEIIIUSrI0WqEEIIIYRodaRIFUIIIYQQrY4UqaLZWCxVjRq3ffYkLp841MTZNE55
+eTmzZ8+mpKSkyeZMTU1l0aJF1+0/evQoV69ebbL1YmJirJ6TWl9VVbX/77d//36OHDnS2LSEEEII
+KzZw7XWn69atY9GiRbz77rscOHDAKqiyspJt27axZMkS/vnPfxIfHy+P3BENUnD+J77++xMtnUat
++vXrx1//+tdaYx555BHatm3bpC+1uOOOOzh48CDvvvtujf0HDx7kH//4R5Ot1xjHjx/nnnvuqTUm
+OTmZ1NTUWmOEEEKI+lIDZGRkMHToUPz8/MjNzWXlypX4+PgQFBQEXLt7ubi4mKeeeorKykrWr19P
+SkoKXbt2bcncxW+I2VDY0inUKiUlBQ8PD7755hvKysrQaDTVYj755BPMZjNz5sxp0rXVajXr1q3j
+rrvuYtSoUXTu3Nmq/+GHH+auu+4iJiYGW1vbJl27vnJzc6/bV1payrx584iLi6OyspKEhAQWL15M
+mzZtmjFDIYQQtxo1wKBBg5SGdu3a4e/vj8lkUtpMJhMBAQHY2tpia2tLhw4d6nyTkbg1/JzyA6e2
+rqOi1ETx1ctEPDabQx+8iYtPIMNfXAFA4YV0Tnz+PnnppygzGvDtMYA+j8/H1s4esz6fhAVRmA0F
+lBmL2PTkSACcvfwZseC/rybN2L+Dk5vXYNbno3Vrx50Tn8S/92Clv7ykmL3/eI7LJw6j8/Rl0F8W
+4vw7f6W/oqKC3r17Ex4eztq1axt8nKtXr+bRRx8lKSmJL774ggcffLBazD/+8Q/WrVvX4Ll/sWjR
+Ivr27Uvfvn2r9Tk6OvK3v/2NZcuWsWzZMqu+Nm3a0Lt3bxISEhg1alSD183Pz2fmzJns2bOH9u3b
+0759e9zd3ZX+lJQU3njjDY4cOUJBQQGjRo0iNjYWBwcHcnJyGDZsGFevXqWwsJDg4GuvvO3QoQO7
+du0C4IMPPuDQoUOcOXMGjUbDmjVrMJlMUqQKIYS4IcqeVIvFQnFxMUlJSZhMJqt3pHfv3p0jR47w
+448/YjQaOXv2LGFhYS2SsGh+2ckHiZj2Au06duPEF2u476315J49iTH3ZwCKLmcR1H8kY97dxLjl
+Oyi8mM5PCRsBcHB1Z8y7m4iMmotnaA/Gx+5kfOxOqwI180A8R/+9mP5PvcoDq3fz+2fepKLUbJXD
+j5/G0vX+ady/dCtat7ac+PwDq36z2Ux6ejqnTp1q8PGVlpayY8cO7r33Xv74xz+yevXqajHZ2dkY
+DIZG/XefnZ0NgNFopKioiKqqKq5cuVIt7g9/+AObN2+ucY5HH320UcU3QFRUFHZ2dmRlZbF161Yu
+Xbpk1X/27FkefPBBjh8/zrlz5zh16hQrV64EwNPTk+PHj7Ns2TIGDBhARkYGGRkZSoH6C5VKhcVi
+Qa1WM2PGDHx8fBqVqxBCCPEL5WH+qampbN++HYvFwpQpU1Cr//ucf1dXV7y9vTl69Cjbtm0jMjIS
+V1fXFklYND9Xv2Dc/Dvg4h2Im38H7F3a4NTOm6IrF3Fq9zv8I+4GoNxkxJCdiYt3IFfPnCC0nvOf
+2vYRPR95Bvfga5e5XX2DcfUNtoq5a8qztG1/bcagfvfw09cbrfp1Oh2ZmZmN2iu6adMmRo4ciUaj
+oXPnzhQXF5Oenk6HDh2UmMzMTKufG2Lt2rVs374ds9nM7t27+fvf/84zzzzDhAkTrOLc3d0pKSmp
+cbvBiBEjeOqpp8jPz7c6C1qXgoICNm/eTG5uLvb29nh4eDB8+HB+/vlnJWbs2LEAFBUVkZaWRseO
+HTl8+HC915g+fTo//vgjQUFBREVFMWfOHPn/ByGEEDdMqURDQ0MJDQ0lPz+fjRs30q9fP2XP6ccf
+f0xkZCSdO3cmPz+f7du3c/DgwRovW4pbl0pV879L8nM4/MGblJtKaHtHGCobWypKius9r+FyFm7+
+tReANr/6o0nr1pbK8rJqMQ0p3n5t4sSJVpf39+/fj42N9YMvysrKsLNr3Ctf582bx6OPPkrPnj0p
+Ly8nOTn5untL1Wp1jUWqra0tDz74IHFxcTz11FP1XjsjIwMPD49aL71funSJ6OhoiouL6dWrF2q1
+Gr1eX+81NBoNq1at4plnnuGtt94iJCSE+Ph47rzzznrPIYQQQvyvao+gcnd3Jzw8nNOnTwPX9qNe
+uXJFuZnD3d2d4cOHc+LEiebNVLRaiYtmEzxgFMNfWkHPh6Px7hZRLcbWTkNpUc03T+k8fdBfyrzh
+PPR6PWVl1YvXuqjVaquiUa1WVytS/fz8uHDhQqPXnz9/PqtXr2bs2LF88MEHNcaYzWaqqqrQ6XQ1
+9td1yb+m9T09PcnPz8dsNl9nFEyaNIlJkyYRHx9PTEwMd999d7UYBwcH8vLyrjsHQFhYGOvWreOR
+Rx5RtgsIIYQQjWVjMpnYsGGD8guooKCAlJQUfH19gWu/nDQaDWlpaVgsFqqqqjh79qxczhOK4quX
+Uf2nqDNcPk9awoZqMW7+HSjMOosx9zIAZkOB0hdyz0SOfrwE/aWM/8yXzcktHzYoB6PRSFBQEIMH
+D27cQdShQ4cOGAwGq8vk9V2/qKiIsLAwxowZw9///ncuX75c4xwJCQkMGzbsujmEhISg0Whq/APx
+euv7+fnRvXt3XnvtNSwWC2fPnmX9+vVWMefPn1eK9DNnztRYYIaFhXHy5EmysrIArJ7bGh0dzYoV
+K8jOzubcuXMcPnyYTp06Xfc4hBBCiPpQa7VaQkJC2LJlC4WFhVgsFsLDw+nTpw9w7YaISZMmkZCQ
+QEJCAhaLBR8fH+69994WTl20FpHTX+D4plUci1uGW0BHQkZMJOvQbqsYnacvPSdHs3P+o9hqHHBq
+583Qee9hY2tLx6HjsFRWsmfhs1SYTTi4tqHbuOkNykGj0RAYGGh1w19TUqlUzJgxg7fffrvGZ5bW
+tr6zszN/+9vfgGuX7V9++eVqMRaLhbfffptXX3211jwee+wxPvzww2o51LZ+XFwc06ZNw8/Pj/Dw
+cKZMmWJ189TSpUuJiYnhxRdfpGvXrjzxxBN88cUXVnMEBwfz+uuvM3DgQLRaLQEBAXz55Zeo1Wpm
+zZrFwoULiYmJwc3NjejoaKZOnVrrcQghhBB1UVlu8Kn8dT2KarmLS639c+pYvq75nZ2dZfxNHC/+
+q7y8nP79+/Pqq6/W+WD7hnrttdc4f/58jU8W+DWDwcCdd97JmTNnrG5ubA3ee+893NzcmDx5ckun
+IoQQ4hYgr0UVop7s7OzYtm0bixcvbtLXop48eZKUlBSWL19eZ6yLiwv9+/fnyy+/bLL1m4q3tzce
+Hh4tnYYQQohbhJxJlfG19gshhBBCtAQ5kyqEEEIIIVodKVKFEEIIIUSrI0WqEEIIIYRodaRIFUII
+IYQQrY4UqUIIIYQQotVRA2RlZbFnzx7y8vJQqVRERkbSr18/JchgMLB9+3auXr2KVqvlnnvuITAw
+sMWSFkIIIYQQtzY1QEZGBkOHDsXPz4/c3FxWrlyJj48PQUFBAHz++ed06dKFhx9+mNzcXP71r38R
+FRUljy8SQgghhBA3hQ3AoEGD8PPzA6Bdu3b4+/tjMpkAMJvNXL58mV69ein9PXv25IcffmihlIUQ
+QgghxK1O2ZNqsVgoLi4mKSkJk8lk9Q7w8vJyysrKlJ89PT25evVq82YqhBBCCCFuG8rLv1NTU9m+
+fTsWi4UpU6Yo7wV3cHDA29ubQ4cOERkZycWLF9m9ezdOTk4tlrQQQgghhLi1KUVqaGgooaGh5Ofn
+s3HjRvr160fXrl0BeOCBB9izZw/r16/Hz8+P3//+95w9e7bFkhZCCCGEELc29f82uLu7Ex4ezunT
+p5Ui1c3Njfvvv1+JiY+Px8vLq/myFEIIIYQQtxUbk8nEhg0byMvLA6CgoICUlBR8fX2VoMzMTEpL
+SwE4d+4cJ0+eVG6kEkIIIYQQoqmptVotISEhbNmyhcLCQiwWC+Hh4fTp00cJunLlCl9++SVlZWW4
+u7szZcoUtFptC6YthBBCCCFuZWqA7t2707179+sGRUZGEhkZ2WxJCSGEEEKI25u8FlUIIYQQQrQ6
+UqQKIYQQQohWR4pUIYQQQgjR6kiRKoQQQgghWh0pUoUQQgghRKsjRaoQQgghhGh1pEgVt5zjm1Zz
+6P03WjoNIYQQQtyAaq9FjYuLo6ioiBkzZihtVVVVfP3116SlpWFra0ufPn246667mjVRIYQQQghx
++7AqUpOTkykvL68WdODAAQwGA3/+858pLS3lww8/xN3dneDg4GZLVAghhBBC3D6UItVgMLBv3z5G
+jRrF7t27rYKOHj3KpEmTsLGxQavV0q9fP44ePSpF6m2iJD+Hb96MZtj8WJLWLiQ7+SBtAjoyYsFq
+AKoqKzm+cRUZ+78CiwXP0J5ETJ2DndZJmSNj/w5Obl6DWZ+P1q0dd058Ev/egwEoKynmhw/fITv5
+ADa2au4Y8ge6jZuGysa2XuuXFuv5flUMP588jLOXHzovP+x1rlb5H4hdgP5COjZ2Gtq2D6PHw0/j
+7OWnxFRUVNC7d2/Cw8NZu3btzf5KhRBCCFEHpUjdtm0bQ4YMwd7e3iqgqqoKg8FAu3bt+O6772jb
+ti2enp4cPny42ZMVLcdUkMu+xc/TafgE+j7+EmUlRUpf8mex5KQe4763P8XOXkvSh29z9OMlRE5/
+AYDMA/Ec/fdi7p7zT9yDO6O/lEF+Rpoy/mDsAuwcdYxb/hXlJiO7Y57C1t6BLqP/WK/1D8a+gq3G
+gQkrEyg3Gfn2nb9aFanJn61A5+nLsLnvAXAh6VurAhrAbDaTnp6ORqNp2i9OCCGEEI1iA3Ds2DHs
+7OwICwurFlBRUYGNjQ0qlYrMzEwuX76MnZ0dpaWlzZ6saDkl+Tl0Gz+dwL7DUTtocXT3VPpSv1pP
+z8mzsHNwBJWKbuOjuJD0rdJ/attH9HzkGdyDOwPg6htM8ICRAJQZizj//S56/ek5bGzV2OtcCX/o
+Kc58vale65cVG8g69A2R057ZCJBNAAAgAElEQVTH1k6Dg0sbfO7sazXWsa0XV04d4edTR6iqqsQ/
+4m4cXNpYxeh0OjIzM9m7d2+TfWdCCCGEaDy1Xq8nMTGRadOm1Rjwy5mliooKJk+eDEBWVhbOzs7N
+lqRoeWoHR37XpXe19lJDAWUmI98te9Gq3d7ZRfm34XIWbv4dapy3OOcSDi5t0DjqlDYX7wCKcy7V
+a/2inEs4uLZBo3Op1veL7uOnY69z5ei/F6PPzsS/1yB6Tp5lVWgDuLu7X3cOIYQQQjQvdVpaGiqV
+ijVr1gDXilGj0ciSJUuIiopCq9Xi6enJpUuXCAwMBODixYt4enrWNq+4Tdg7u2Hn4Mjwl1bg1M67
+xhidpw/6S5m0CexUrc/JwxuzoYByk1G5BF905SJOHj71Wl/r6k5pkZ7K8lJs7exrjFHZ2NJ51EN0
+HvUQpcV6Dq1+nYMrX2XoC0ut4vR6PVqtVi75CyGEEK2ATUREBNHR0cpn4sSJeHl5ER0djVarBaBX
+r17s3buXyspKiouLSUpKomfPni2cumgVVCpCRjzA96tiKCspBsCszyc/I1UJCblnIkc/XoL+UgYA
+xVezObnlQwDsda4ERAzhyL/exVJVSVlJMT9+spyOw8bVa3nHtl60CerE8Q2rwGKh6OcsMhK/soo5
+un4JhRfSr63n5IKrX3uwWKxijEYjQUFBDB48uDHfghBCCCGaWLXnpNYkPDycwsJCYmNjsbGxYdiw
+YXh5ed3s3MRvRI/J0ZzY9D5fPT8ZVCo0jjq6T5ih7EHtOHQclspK9ix8lgqzCQfXNnQbN10Z32/m
+KyStXcimJ/8PG1tbOgweTZfRU+q9/sBn3+LAey+zYcYw3IM6037wfZTk5Sj9Hnd0I2ntQoqvXsZS
+VYWLTwB9ouZbzaHRaAgMDKRjx443+G0IIYQQoimoLJb/OaXUQEVFRbX2L3e5/l5BgDl1LF/X/HXt
+jZXxNzZeCCGEEKIlyGtRhRBCCCFEqyNFqhBCCCGEaHWkSBVCCCGEEK1OvW6cErcv2dMqhBBCiJYg
+Z1KFEEIIIUSrI0WqEEIIIYRodaRIFc3u+KbVHHr/jQaPs1iqbmjd/Mw0Pps2hNyzJ5W2ilIzm54c
+yaYnR/LRg3dx+cShG1rjt6Cx378QQgjRnKrtSY2Li6OoqIgZM2YobUajkUOHDpGamoqLiwuPPPJI
+syYpRMH5n0j68B1GvLyqWl/2jwfYFTMTje6/z+S1s9cyfkW8VZxjGw8C+w7Dqe1/X0ShtndgfOxO
+ALbPntSo3Ha/8TRXf0pGrdFSYS7Bo1N3ImfMQ1fPV7sKIYQQojqrIjU5OZny8vJqQTY2Nvj4+FBW
+VkZubm6zJSfEL8yGwlr72wR2ZPQ7G2qNcXB1J3L63KZMS9H7T8/RYfAYykuKOf75+3y/6jWGzVt+
+U9YSQgghbgdKkWowGNi3bx+jRo1i9+7dVkFarZbOnTtLkSoapbRYz/erYvj55GGcvfzQeflhr3NV
++gsvpHPi8/fJSz9FmdGAb48B9Hl8PrZ29pj1+SQsiMJsKKDMWMSmJ0cC4Ozlz4gFq+u1fsKCKIqu
+XADAmPszYxZtws2/Q73zr6qs5PjGVWTs/wosFjxDexIxdQ52WqdqsXaOOoL6Didz/44Gjc/Yv4OT
+m9dg1uejdWvHnROfxL/3YADKSor54cN3yE4+gI2tmjuG/IFu46ahsrEFoCQ/h2/ejGbY/FiS1i4k
+O/kgbQI6Kt9PXd8/QEVFBb179yY8PJy1a9fW+7sRQgghbhalSN22bRtDhgzB3t6+JfMRt6CDsa9g
+q3FgwsoEyk1Gvn3nr1ZFUtHlLIL6j6T/n1+lqqKC+Jen8lPCRkLvnYyDqztj3t3E+e93kRb/WY2X
+++vy62L2s2lDGjw++bNYclKPcd/bn2JnryXpw7c5+vESIqe/UC221FDAmV1f4N4+rN7jMw/Ec/Tf
+i7l7zj9xD+6M/lIG+RlpyviDsQuwc9QxbvlXlJuM7I55Clt7B7qM/qMSYyrIZd/i5+k0fAJ9H3+J
+spKiX42v/fsHMJvNpKeno9FoGvz9CCGEEDeDDcCxY8ews7MjLCysrnghGqSs2EDWoW+InPY8tnYa
+HFza4HNnX6sY/4i78e81iMqyUvQX03HxDuTqmRMNWqfg/Bk+eWyg8rnww94mO4bUr9bTc/Is7Bwc
+QaWi2/goLiR9axWTtO4dNj5+D59MHUxVVQX9nny53uNPbfuIno88g3twZwBcfYMJHnDtjHGZsYjz
+3++i15+ew8ZWjb3OlfCHnuLM15us1i/Jz6Hb+OkE9h2O2kGLo7vntfH1+P4BdDodmZmZ7N3bdN+b
+EEIIcSPUer2exMREpk2b1tK5iFtQUc4lHFzbWN3U9L9K8nM4/MGblJtKaHtHGCobWypKihu0Tn32
+pDZGqaGAMpOR75a9aNVu72x9PL3/9BztB97LlmfG4XNnP+yd3eo93nA567rbD4pzLuHg0gaNo05p
+c/EOoDjnklWc2sGR33XpXW18fb7/X7i7u9cZI4QQQjQXdVpaGiqVijVr1gDX9qYZjUaWLFlCVFQU
+Wq22hVMUv2VaV3dKi/RUlpdia1fzVpLERbMJvXcygX2HA5D+7VayDn1jFWNrp6G0qPabp26UykaF
+pbLSqs3e2Q07B0eGv7QCp3bedYy3JfyhmRyLW0pA5FBsbG3rNV7n6YP+UiZtAjtV63Py8MZsKKDc
+ZFT2sBZduYhTPZ8cUJ/v/xd6vR6tViuX/IUQQrQKNhEREURHRyufiRMn4uXlRXR0tBSo4oY5tvWi
+TVAnjm9YBRYLRT9nkZH4lVVM8dXLqGyuPbLXcPk8aQnVz4i6+XegMOssxtzLAJgNBU2eq87Dh4tH
+94HFQmmx/lqjSkXIiAf4flUMZf85u2vW55OfkVrjHEF9R2CrcSB9z5Z6jw+5ZyJHP16C/lIGAMVX
+szm55UMA7HWuBEQM4ci/3sVSVUlZSTE/frKcjsPG1euY6vP9w7XHzAUFBTF48OB6zSuEEELcbNWe
+k1qTjRs3cvHiRcrKyigrK2Px4sW4uLgwderUm52fuAUMfPYtDrz3MhtmDMM9qDPtB99HSV6O0h85
+/QWOb1rFsbhluAV0JGTERLIOWT9hQufpS8/J0eyc/yi2Ggec2nkzdN572NjaNlme3SfMYO+i2Wx4
+fAReYXcx8Jk3AegxOZoTm97nq+cng0qFxlFH9wkzlD2kVlQqejz0FN+vjqH9oHuxtbOvc3zHoeOw
+VFayZ+GzVJhNOLi2odu46cqU/Wa+QtLahWx68v+wsbWlw+DRdBk9pd7HVdf3D6DRaAgMDKRjx46N
++OaEEEKIpqeyWCyWG5mgqKio1v7lLrXvhZtTx/J1ze/s7CzjW/F4IYQQQojGkNeiCiGEEEKIVkeK
+VCGEEEII0epIkSqEEEIIIVqdet04JURjyZ5WIYQQQjSGnEkVQgghhBCtjhSpQgghhBCi1ZEiVdzS
+KkrNbHpyJJueHMlHD97F5ROHWjql60pOTsbb25ukpCSlraSkhODgYIKDg7G3t2f37t21zCCEEELc
+OqrtSY2Li6OoqIgZM2YobZcuXWL37t1cvXoVe3t7hg4dSmhoaLMmKkRjqO0dGB+7E4Dtsye1SA7x
+8fHce++9tGnTRmlzcnIiMzPTKs7b25vx48fj5+entDk6OpKRce1NVL17926WfIUQQojWwKpITU5O
+pry83CrAYrGQmJjI0KFD8fHx4dy5c8TFxTFr1iy56UWIeurWrRvHjh2rNcbT05Nly5Y1U0ZCCCFE
+66Zc7jcYDOzbt4/+/ftbBahUKiZNmoSvry8qlYoOHTrg5eXFlStXmj1ZcXsqyc9h++yHMBsK2PfP
+F/h06mASFkQp/WUlxRxYvoCNj4/g85n/x/GNq7BUVdZ7/qrKSn78NJYvnh7NF3++j+/ee4lyk9Eq
+pqKigh49evDYY4811WEphg0bplzSV6vVpKSkNGh8RUUFCxYsICQkhE6dOjF16tQ6n6oghBBCtHZK
+kbpt2zaGDBmCvb19rQOqqqooKCigXbt2Nz05IX5hKshl3+LnCYgYwvjlOxgQHaP0HYxdACoYt/wr
+7l0Yx8UjiZz68uN6z538WSxXTv3AfW9/yv1Lt6Fx1HH04yVWMWazmfT0dE6dOtVUh6TYtWsXGRkZ
+ZGRk4OHh0eDxr7zyComJiRw5coS0tDRcXV2ZO3duk+cphBBCNCcbgGPHjmFnZ0dYWFidAw4ePEj7
+9u1xc3O76ckJ8YuS/By6jZ9OYN/hqB20OLp7AlBmLOL897vo9afnsLFVY69zJfyhpzjz9aZ6z536
+1Xp6Tp6FnYMjqFR0Gx/FhaRvrWJ0Oh2ZmZns3bu3UfmfOHECDw8P5bNt27ZGzVOTpUuX8vrrr6PT
+6VCpVMydO5etW7c22fxCCCFES1Dr9XoSExOZNm1ancEZGRkcOXKEqVOnNkNqQvyX2sGR33WpfuNQ
+cc4lHFzaoHHUKW0u3gEU51yq17ylhgLKTEa+W/aiVbu9s0u1WHd39wZm/V/12ZPaGLm5uRgMhmrb
+EG4kVyGEEKI1UKelpaFSqVizZg1wbX+b0WhkyZIlREVFodVqAcjOzmbz5s08/PDD6HS62uYUotk4
+eXhjNhRQbjJip3UCoOjKRZw8fKrFqmxUWCqt96raO7th5+DI8JdW4NTOu9a19Ho9Wq0WjUbTdAfQ
+ADY2NlRUVFi1tW3bFp1OR3x8PAEBAS2SlxBCCHEz2ERERBAdHa18Jk6ciJeXF9HR0UqBeuHCBT79
+9FOlT4jWwl7nSkDEEI78610sVZWUlRTz4yfL6ThsXLVYnYcPF4/uA4uF0mL9tUaVipARD/D9qhjK
+SooBMOvzyc9ItRprNBoJCgpi8ODBN/uQriswMJAdO3ZgsVjIz88Hrt3Y+MQTTzBz5kz0+mvHlJOT
+c1PO2gohhBDNqc6H+ZeXl/PRRx9hNpv55JNPWLhwIQsXLuTf//53c+QnRJ36zXyFyvJSNj35f2x/
+biI+d/ahy+gp1eK6T5hBdvJBNjw+gkPvv6G095gcTbs7uvLV85PZPOsPfPNmNCX5OVZjNRoNgYGB
+dOzY8aYfz/XMnz+fhIQEAgICePrpp5X2mJgYIiIi6NOnD2FhYYwdO5bs7OwWy1MIIYRoCiqLxWK5
+kQnqetTNcpfqe/t+bU4dy9c1f13PapXxv+3xQgghhLg9yWtRhRBCCCFEqyNFqhBCCCGEaHWkSBVC
+CCGEEK2OFKlCCCGEEKLVkSJVCCGEEEK0OlKkCiGEEEKIVkeKVHHbsFiqamzfPnsSl08cuunr79+/
+nyNHjtQZFxMTY/Uc1Pqqqqr5+OorOTkZb29vkpKSlLaSkhKCg4MJDg7G3t6e3bt339AaQgghRH1V
+K1Lj4uJYtWqVVVtWVhbr1q1j0aJFvPvuuxw4cKDZEhSiKRSc/4mv//5Ei+aQnJxMampq3YGNcPz4
+ce65554a++Lj41Gr1Xh4eCifoKCganHe3t6MHz8ePz8/pc3R0ZGMjAwyMjLo3r37TcldCCGEqIn6
+1z8kJydTXl5eLSgjI4OhQ4fi5+dHbm4uK1euxMfHp8ZfdEK0RmZDYYutXVpayrx584iLi6OyspKE
+hAQWL15MmzZtmmyN3NzcWvu7detW56tSPT09WbZsWZPlJIQQQtwIpUg1GAzs27ePUaNGVbukN2jQ
+IOXf7dq1w9/fH5PJ1HxZitva9tmTCLvvEdLiP8Nw+TweHbvR/6m/Y+9yrcgrvJDOic/fJy/9FGVG
+A749BtDn8fnY2tlj1ueTsCAKs6GAMmMRm54cCYCzlz8jFqxW1igvKWbvP57j8onD6Dx9GfSXhTj/
+zl/pr6iooHfv3oSHh7N27doG5f/BBx9w6NAhzpw5g0ajYc2aNZhMJqVIzc/PZ+bMmezZs4f27dvT
+vn173N3dlfEpKSm88cYbHDlyhIKCAkaNGkVsbCwODg7k5OQwbNgwrl69SmFhIcHBwQB06NCBXbt2
+1Su/YcOGkZ6eDsCFCxdITk6mS5cu9T6+iooKXnvtNeLi4rBYLAwYMIB//vOf8jYxIYQQN0S53L9t
+2zaGDBmCvb19jYEWi4Xi4mKSkpIwmUwt+g5zcftJ/3Yrg597hwdWfY2N2o4fPnpX6Su6nEVQ/5GM
+eXcT45bvoPBiOj8lbATAwdWdMe9uIjJqLp6hPRgfu5PxsTutClSAHz+Npev907h/6Va0bm058fkH
+Vv1ms5n09HROnTrVqPxVKhUWiwW1Ws2MGTPw8fFR+qKiorCzsyMrK4utW7dy6dIlq7Fnz57lwQcf
+5Pjx45w7d45Tp06xcuVK4NrZz+PHj7Ns2TIGDBigXJqvb4EKsGvXLmWch4dHg4/tlVdeITExkSNH
+jpCWloarqytz585t8DxCCCHEr6kBjh07hp2dHWFhYVy8eLHGwNTUVLZv347FYmHKlCmo1eoa44S4
+GbrePxVtm2sFVIe7x3Iw9hWlzz/ibgDKTUYM2Zm4eAdy9cwJQhsw/11TnqVt+2sjgvrdw09fb7Tq
+1+l0ZGZm4ujo2ODcp0+fzo8//khQUBBRUVHMmTMHV1dXAAoKCti8eTO5ubnY29vj4eHB8OHD+fnn
+n5XxY8eOBaCoqIi0tDQ6duzI4cOHG5TDiRMnrArQNWvWMHr06AYfS02WLl3Kzp070el0AMydO5de
+vXqxdOnSJplfCCHE7Umt1+tJTExk2rRptQaGhoYSGhpKfn4+GzdupF+/fnTt2rWZ0hTiv9z8O1Ba
+rFd+LsnP4fAHb1JuKqHtHWGobGypKClu0Jw2v/qjS+vWlsrysmoxv74E3xAajYZVq1bxzDPP8NZb
+bxESEkJ8fDx33nmncvaytv2ply5dIjo6muLiYnr16oVarUav1183vib12ZPaGLm5uRgMBh577DGr
+9sZ+V0IIIcQv1GlpaahUKtasWQNc219mNBpZsmQJUVFRaLVaqwHu7u6Eh4dz+vRpKVJFiyi6nIXO
+01f5OXHRbELvnUxg3+HAta0BWYe+sRpja6ehtOjGbp7S6/VotVo0Gk2jxoeFhbFu3Tqee+45Vq5c
+yfLly/H09CQ/Px+z2YyDg0ON4yZNmkR0dDQTJkwAYN26dWzevNkqxsHBgby8vEblVV82NjZUVFRY
+tbVt2xadTkd8fDwBAQE3dX0hhBC3F5uIiAiio6OVz8SJE/Hy8iI6OhqtVovJZGLDhg3KL8CCggJS
+UlLw9fWtY2ohmk7mgQQqy0spKykm+bMVdBx6v9JXfPUyKptr26sNl8+TlrCh2ng3/w4UZp3FmHsZ
+ALOhoEHrG41GgoKCGDx4cINzj46OZsWKFWRnZ3Pu3DkOHz5Mp06dAPDz86N79+689tprWCwWzp49
+y/r1663Gnz9/HltbWwDOnDmj7Ef9tbCwME6ePElWVhYAV69ebXCedQkMDGTHjh1YLBby8/OBa3tt
+n3jiCWbOnKmc3c3JybkpZ22FEELcXup8mL9WqyUkJIQtW7awaNEi1qxZQ0BAAH369GmO/IQAQG3v
+wJezH2bLrLF4dA4nbPQUpS9y+guc+OIDtjxzP8fi3iNkxMRq43WevvScHM3O+Y+yOXos+xa/QFVl
+Zb3X12g0BAYGNuqGwVmzZnHs2DEiIyMZO3YsU6ZMsXpYf1xcHImJifj5+TFr1iymTJliNX7p0qW8
++eabdO3alRdffJEnnqj+vNfg4GBef/11Bg4cSGhoKI888ki1s543av78+SQkJBAQEGCVf0xMDBER
+EfTp04ewsDDGjh1LdnZ2k64thBDi9qOyWCyWG5mgqKio1v7lLi619s+pY/m65q/rMTcy/rc9Hq49
+guquKc/i3S2iztjW7L333sPNzY3Jkye3dCpCCCFEqye36IvfiBv6W6pV8Pb2Vu6AF0IIIUTtpEgV
+opmMGzeupVMQQgghfjOkSBWt3n0L41o6BSGEEEI0szpvnBJCCCGEEKK5SZEqhBBCCCFaHSlShRBC
+CCFEqyNFqvjNMOZeJuGVGXw27W62/vUBsn88YNWfn5nGZ9OGkHv2pNJWUWpm05Mj2fTkSD568C4u
+nzjU3Gkr9u/fz5EjR+qMi4mJsXoOaX1VVVU1Ji1FcnIy3t7eJCUlKW0lJSUEBwcTHByMvb09u3fv
+vqE1hBBCiPqqVqTGxcWxatWq6w6oq1+Im+WHdYtw9Q1i/Iqd3PvGv/EM7WnV79jGg8C+w3Bq66W0
+qe0dGB+7k/GxO2kT2Km5U7aSnJxMamrqTZn7+PHj3HPPPTX2xcfHo1ar8fDwUD5BQUHV4ry9vRk/
+fjx+fn5Km6OjIxkZGWRkZNC9e/ebkrsQQghRE6u7+5OTkykvL79ucF39QtxMBed/ov+fX8XWzr7G
+fgdXdyKnz23mrOpWWlrKvHnziIuLo7KykoSEBBYvXkybNm2abI3c3Nxa+7t161bnq0o9PT1ZtmxZ
+k+UkhBBC3AjlTKrBYGDfvn3079+/xsC6+oW4WX741yK+eHoMhp+z+PbtZ9n05EgSFkQp/QkLopRL
++v96IJzCC+kNmr+qspIfP43li6dH88Wf7+O7916i3GS0iqmoqKBHjx489thjDc7/gw8+4NChQ5w5
+c4aLFy/Sv39/TCaT0p+fn89DDz2El5cXffv25dSpU1bjU1JSeOSRRwgNDeV3v/sdjz32GGazGYCc
+nBy6d+/O5MmT2b9/v3JpftiwYfXOb9iwYco4tVpNSkpKg46voqKCBQsWEBISQqdOnZg6dWqdbxoT
+Qggh6qIUqdu2bWPIkCHY29d8lqqufiFull5//Av3L92KzsOXofOWMz52JyMWrFb6RyxYrVzSd3Bx
+b/D8yZ/FcuXUD9z39qfcv3QbGkcdRz9eYhVjNptJT0+vVkDWl0qlwmKxoFarmTFjBj4+PkpfVFQU
+dnZ2ZGVlsXXrVi5dumQ19uzZszz44IMcP36cc+fOcerUKVauXAlcO/t5/Phxli1bxoABA5RL87t2
+7ap3brt27VLGeXh4NPjYXnnlFRITEzly5AhpaWm4uroyd27rO6MthBDit8UG4NixY9jZ2REWFlZj
+UF39QvyWpX61np6TZ2Hn4AgqFd3GR3Eh6VurGJ1OR2ZmJnv37m3w/NOnT6dz584EBQUxd+5c9Hq9
+0ldQUMDmzZtZsmQJ9vb2eHh4MHz4cKvxY8eOZfTo0ZjNZk6dOkXHjh05fPhwg3I4ceKE1Z7Ubdu2
+Nfg4rmfp0qW8/vrr6HQ6VCoVc+fOZevWrU02vxBCiNuTWq/Xk5iYyLRp02oMqKtfiN+yUkMBZSYj
+3y170ard3tmlWqy7e8PP0gJoNBpWrVrFM888w1tvvUVISAjx8fHceeedytnL2vanXrp0iejoaIqL
+i+nVqxdqtdqq0K2P+uxJbYzc3FwMBkO1bRCN/a6EEEKIX6jT0tJQqVSsWbMGuLa/zGg0smTJEqKi
+oqirX4jfCpWNCktlpVWbvbMbdg6ODH9pBU7tvGsdr9fr0Wq1aDSaRq0fFhbGunXreO6551i5ciXL
+ly/H09OT/Px8zGYzDg4ONY6bNGkS0dHRTJgwAYB169axefNmqxgHBwfy8vIalVd92djYUFFRYdXW
+tm1bdDod8fHxBAQE3NT1hRBC3F5sIiIiiI6OVj4TJ07Ey8uL6OhotFotdfUL8Vuh8/Dh4tF9YLFQ
+WvyfM5EqFSEjHuD7VTGUlRQDYNbnk59h/agoo9FIUFAQgwcPbvC60dHRrFixguzsbM6dO8fhw4fp
+1Ona47D8/Pzo3r07r732GhaLhbNnz7J+/Xqr8efPn8fW1haAM2fOKPtRfy0sLIyTJ0+SlZUFwNWr
+VxucZ10CAwPZsWMHFouF/Px84Npe2yeeeIKZM2cqZ3dzcnJuyllbIYQQtxd5mL+4bXSfMIPs5INs
+eHwEh95/Q2nvMTmadnd05avnJ7N51h/45s1oSvJzrMZqNBoCAwPp2LFjg9edNWsWx44dIzIykrFj
+xzJlyhSrh/XHxcWRmJiIn58fs2bNYsqUKVbjly5dyptvvknXrl158cUXeeKJJ6qtERwczOuvv87A
+gQMJDQ3lkUceqXbW80bNnz+fhIQEAgICrPKPiYkhIiKCPn36EBYWxtixY8nOzm7StYUQQtx+VBaL
+xXIjE9T1qJnlLtX39v3anDqWr2t+Z2dnGX8Lj7+VvPfee7i5uTF58uSWTkUIIYRo9dR1hwghmoK3
+tzc6na6l0xBCCCF+E6RIFaKZjBs3rqVTEEIIIX4zZE+qEEIIIYRodeRMqripbqc9p0IIIYRoOnIm
+VQghhBBCtDpSpAohhBBCiFZHilQhfiMslqqWTkEIIYRoNtX2pMbFxVFUVMSMGTOUtuTkZLZu3Yqd
+nZ3SNnr0aLp06dI8WYrfrNKiQj55dCB9Zswj5J4HAdi/ZB45acmMe297C2f321Hw/+3de1BUV57A
+8W9DQ9PSAoJAEKFBJQoqEYKoo5MYokbjOIxi1AxaGaM4mq00MymKRGIm6kaTGGp11PgcfOyuYSer
+jovRqFEnojFRoyyorIw8hAhGRaB52Tx7/2C8sQNC4wNRf5+qrqLu+f3O/d0rlKdPn3s6/x+c2pLI
+mPc3POxShBBCiA5hMUhNT0+nrq6uWZDJZCIsLIxx48Z1WGHi8aHp6sKl4wfo+9JUGuvruH7x7MMu
+6ZFjKi972CUIIYQQHUoZpJaXl3P06FHGjRvHoUOHLIJu3ryJo6NjhxcnHg/2jk6YjCWYjCUUZ5/D
+2duPsh9ylfbGhgYytm8g79heMJvxCAwl/PW3sdM2/c6V/ZDD2Z1/4UZOJrVV5XiHjGDo7xdga6cB
+oLrkGsfXLsT4Qw42dva49Qoi5Ldv0tWzJwBbo4KZtiUVTVcXANKSV1NvqmbwzHgl//BHBkYtWMup
+zcsoSv+Wbr4BjFm4sc36fjz/PZkpW6mvuUnl9SuEz4znRNJHOPXQM/q9dVZd3xfxrzJs7p8497ck
+rpw9ic7Dm+ffWkbXp71JRsEAAA8ySURBVHwwGUs4sDAGU3kptVUV7Jg3FoCunj5KfQD19fUMHjyY
+QYMGsXnz5gf2bymEEEJ0FGVN6u7du4mIiECj0TQLMplMFBQUkJyczLZt2zhz5kyHFikebfWmavRD
+R1Fw4hD5335Fj0HDLdrTP1/L1czv+dUnf2Xiqt3Yd9FxZttKpb3iSgF+w8fy6+U7mLTmS8ou5/CP
+A9tvy1+HzsObqLX7mLgyBf/hY5UBoLVulhZzdMU7+IZHELXmS0YYllhdX1H6t4TPmk/3gIGc/dsm
+fvXxZxRnn6Oq+Eer8gG+XbuQARNnMXFVCloXN87uTALAwdmVXy/fwZCYBDwCQ4hau4+otfssBqjQ
+9Deak5NDZmZmu65bCCGE6KxsANLS0rCzsyMoKKjFoP79+xMeHk5UVBQREREcO3aMtLS0Di1UPLoa
+6mrwf248BScPU3IpC4++z1i0X9j7GaHRsdg5dAGVioFRMfxw6mul3Sf8BXzCnqehtgbj5RycvPQW
+Swa6uHlyNfM0P2aeprGxAZ/wF3Bw6tauGqtLrjEwajb6YaNRO2jp4uphdX3OPf1x8emNk5ce79AR
+aJy64djdi4qrl63KBwj5rQG3XoFourrg94uXMBbmtat+nU7HpUuXOHLkSLvyhBBCiM5KbTQaSU1N
+ZdasWXcM8vHxUX728vJi+PDhXLhwgZCQkI6oUTwGnHv4YSovxTtkBKhUyvGa8lJqb1bxzer3LOI1
+XZ2Un6tLrnEy6SPqblbj1icIlY0t9dWVSntw1Gw0OmfO/OcKjEWX8Al7ntDoWIuBZlvUDl14qv/g
+Zsetqe+W2y5L+dnafBv1T8vDtS5uNNTVWl37La6uru3OEUIIITordVZWFiqVik2bNgFNa9uqqqpY
+uXIlMTExaLXaZkkqlQobG9m9SrTP828tw17nrHwMDk0PVdk5dGH0n9bh2N2rxbzUf4sncHw0+mGj
+Acj5OoWCE4eVdpWNLf3GTaPfuGnUVBo5sXEp367/V16cvwoAG7UdpvJSZU1qY33zhwPvxJr6HmT+
+LbZ29tRUtP7wlNFoRKvVYm9vf9fnEUIIIToLm/DwcAwGg/KaMmUKnp6eGAwGtFotVVVVbN++ndLS
+UgDKysr45ptvCAwMfMili0dN16d80eicLQ+qVPQd8wrfbVhC7T9nR03GEkryLighldevoPrnm6Ly
+K/lkHfhviy7OfLaSsh9yANA4OuHcsxeYzUq7Uw89OV/vpqGuhsvfHyE3tR1bX1lR3wPN/ycXn96U
+FWRTVXylqY/yUov2qqoq/Pz8GDlyZLv6FUIIITqrZvuk/pyjoyN9+vRh586dVFRUYGNjw5AhQwgO
+Du6I+sQTICTawNkdf2HvO9GgUmHfRUfw5Dm4+vcDYMjs+WTs2EBa8mpcfAPoO2YKBSd+2oHCvc9A
+Tm1eRuX1K5gbG3Hq4cvQmAVKe/jMeI6vXUjO1ynoh40mNDq2XYPEtup70PkAOg9vQqMN7FvwO2zt
+HXDs7sWL736Kja0tAPb29uj1egICAqzuUwghhOjMVGbzbVNOd6GioqLV9jVOzdfu3e7tNk7fVv9d
+u3aV/E6cL4QQQghxN2RhqRBCCCGE6HRkkCqEEEIIITodGaQKIYQQQohOp80Hp4S4F7KmVQghhBB3
+Q2ZShRBCCCFEpyODVCGEEEII0enIIFU8Mczmxke6/7YcO3aM06dPtxm3ZMkS3nzzzXb339h4b9eX
+np6Ol5cXp06dUo5VV1fj7++Pv78/Go2GQ4cOtdLD4+Fu778QQjxpmg1Sk5OT2bBhQ7PA7Oxs1q9f
+T2JiIhs3biQnJ6dDChTifijN/wdfLZ77yPZvjfT0dC5caN83WVkrIyODl156qcW2/fv3o1arcXd3
+V15+fn7N4ry8vIiKiqJnz57KsS5dupCXl0deXt5df0HIhAkTcHd3R6/X4+bmxvjx48nPz7+rvoQQ
+QnQeFg9OpaenU1fX/HvNi4qK2LNnD9OmTcPT05MbN25QU1PTYUUKca9M5a1/731n7x/g3LlzdOvW
+DW9vb4vjNTU1vPvuuyQnJ9PQ0MCBAwdYsWIF3bp1u2/nLi4ubrV94MCBpKWltRrj4eHB6tWr71tN
+t0tMTOS1116jvLycpUuXMm/ePPbu3ftAziWEEKJjKDOp5eXlHD16lOHDhzcLSk1NJSIiAk9PTwDc
+3Nzo0aNHx1Upnmi11ZUcX7OQ7b8fw843XiZj+wbMjQ1K+9aoYGoqfhokpiWv5tTmZQCYjCWk/DGK
+oyve4dr/pbFj3lh2zBvLgYUxSvwX8a+Sm7qHL999jb++PpLDH75JTXnpfesfoL6+npCQEGbOnHnX
+92Hbtm189913zY4nJSVx4sQJLl68yOXLlxk+fDg3b95U2ktKSpQ3mMOGDSMzM9Mi//z580yfPp3A
+wECeeuopZs6ciclkAuDatWsEBwcTHR3NsWPHlI/mR40aZXXdo0aNUvLUajXnz59v13XX19ezcOFC
++vbty9NPP83rr79+x10jnJycmDx5ssU1WpOfnJxMSEgI3t7ePPvss6SkpChtRqOR2bNn4+vrS+/e
+vfnggw9oaPjp96+wsJCwsDCuX7/O9OnT8fT0tLg/bd1/IYQQLVMGqbt37yYiIgKNRtMs6OrVq7i4
+uLBnzx62bNnCoUOHqK2t7dBCxZPr27ULQQWT1uxl/LJkLp9OJXPPNqtyHZxd+fXyHQyJScAjMISo
+tfuIWruPMQs3WsTlfJ3CyLhEXtnwFTZqO77/j+X3tX+TyUROTs4DG6CoVCrMZjNqtZo5c+ZYvImM
+iYnBzs6OgoICUlJSKCwstMjNzs5m6tSpZGRkkJubS2ZmJuvXrweaZj8zMjJYvXo1I0aMUD6aP3jw
+oNW1HTx4UMlzd3dv97UtWrSI1NRUTp8+TVZWFs7OziQkJLQYW1xcTFJSEqGhoVbnf/7558yfP5/N
+mzdTWFjItm3bqK6uVtpjYmJQqVTk5ORw6tQp9uzZw5///GeL8/74449ER0fzm9/8htzcXLZu3WqR
+39r9F0II0TIbgLS0NOzs7AgKCmoxqKKigsOHDxMaGsrUqVO5cePGE/GAg3j4aqsqyP/uIGGvxWFj
+q0ajc2bQtH/h4lc77ut5Bkx8HW03d2zUdvR+IZLCM8fua/86nY5Lly5x5MiRdueOHTuWwYMHs2XL
+FuLj4xk8eDBxcXFK++zZs+nXrx9+fn4kJCRgNBqVttLSUnbt2sXKlSvRaDS4u7szevRoi/4jIyOZ
+MGECJpOJzMxMAgICOHnyZLtqPHv2rMWa1N27d7f7Ou9k1apVLF26FJ1Oh0qlIiEhwWKmEyAuLg69
+Xo+Hhwf19fVs3LjR6vzly5fz4YcfMmjQIAD69evHtGnTACgrK2Pnzp0kJiZiZ2eHq6srixcvtugf
+mmZTExISmDx5Mo6OjsqSDGvuvxBCiJapjUYjqampzJo1645Bjo6OREZG4uLiAsCQIUOa/SchxINQ
+ea0QB6du2HfRKcecvHypvPbgZqNcfHpTU2lsO7CdXF1d7ypv3759AMyfP5+wsDCioqIs2u3t7dmw
+YQN/+MMf+Pjjj+nbty/79+/nmWeeUWYvW1ufWlhYiMFgoLKykrCwMNRqtcVA1xrWrEm9G8XFxZSX
+lzdbJvHze5mYmMj06dMZMGAAY8aMwc3Nzer8ixcv0r9//xbPn5eXR/fu3XF2dlaO9enTh7y8PIs4
+nU7HyJEjW8xv6/4LIYRomTorKwuVSsWmTZuApvVbVVVVrFy5kpiYGLRaLe7u7hQXFyuDVJ1O11qf
+Qtw3ju5emMpLqbtZhZ3WEYCKq5dxdP/p42wbtR2m8lI0XZt+Pxvrmz/8Z2tnb7GutDUVVwrQefz0
+cNL96t9oNKLVarG3t7eqjvYKCgpi69atxMXFsX79etasWYOHhwclJSWYTCYcHBxazHv11VcxGAxM
+njwZgK1bt7Jr1y6LGAcHB27cuPFA6r7FxsaG+vp6i2Nubm7odDr279+Pr69vq/m2trYsWrSIBQsW
+MHHiRNRqtVX5fn5+ZGVltbi7gF6vp7i4mIqKCuXb0XJzc1vcvaAl1tx/IYQQLbMJDw/HYDAorylT
+puDp6YnBYECr1QJNM6d///vfMZlMmM1mjh8/ztNPP/2QSxdPAo3OGd/wCE7/+3LMjQ3UVlfyv/+1
+hoBRk5QYpx56cr7eTUNdDZe/P0Ju6hfN+nHx6U1ZQTZVxVcAMN32YBTApeMHaKiroba6kvTP1xHw
+4sT72n9VVRV+fn4tzrZZ65e//GWLf3cGg4F169ZRVFREbm4uJ0+eVOJ69uxJcHAwH3zwAWazmezs
+bD777DOL/Pz8fGxtbYGmWcVb61FvFxQUxLlz5ygoKADg+vXrd30dd6LX6/nyyy8xm82UlJQATWtt
+586dyxtvvKHM7l67du2Os7avvPIKWq2WLVu2WJ0/b948EhISlO278vPz+eSTT4CmGdfIyEji4+Np
+aGjAaDTy/vvvt/rJ0+2suf9CCCFaZtVm/gEBAYSGhrJp0yZlC5mIiIgHWpgQt/zijUU01NWwY97L
+fBE3hR7PDKX/hBlKe/jMePKO7WXnG+MpyviO0OjYZn3oPLwJjTawb8Hv2GWI5OiK+TTe9oS2WuPA
+nvjf8j+xkbj3G0TQfe7f3t4evV5PQEDAXd+Hl19+mYEDBzY7HhsbS1paGkOGDCEyMpIZM2ZYbBaf
+nJxMamoqPXv2JDY2lhkzZljkr1q1io8++ogBAwbw3nvvMXdu8/1e/f39Wbp0Kc899xyBgYFMnz69
+2aznvVqwYAEHDhzA19fXov4lS5YQHh7O0KFDCQoKIjIykqKiohb7UKlULF68mEWLFik7FLSVP2vW
+LOLi4pg0aRJ6vZ6oqCh69eqltCclJWEymejVqxehoaGMHj2at956y+rrauv+CyGEaJnKbDab76WD
+O20Fc8saJ6dW299u4/Rt9X/rIzjJfzzzO8IX8a/y7Iw/4jUw/GGXck8+/fRTXFxciI6OftilCCGE
+EPdM3XaIEE+Ce3qv1il4eXnJenEhhBCPDRmkCvGYmDRpUttBQgghxCNCBqniiferZckPuwQhhBBC
+/MwDX5PaGdYcCiGEEEKIR4tVT/cLIYQQQgjRkWSQKoQQQgghOh0ZpAohhBBCiE6n2YNTycnJVFRU
+MGfOHKDpm3JWrVplEdPQ0IBOpyM2tvmm5kIIIYQQQtyr/wcr3UlLfH/DGgAAAABJRU5ErkJggg==
+"
+ id="image817"
+ x="2.5102806"
+ y="0.015830245" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect517"
+ width="73.620293"
+ height="15.372134"
+ x="2.1647058"
+ y="5.0289979" /><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="99.639122"
+ y="9.133729"
+ id="text1127"><tspan
+ sodipodi:role="line"
+ id="tspan1125"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="99.639122"
+ y="9.133729">Converter-Node on level 0 (root):</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="99.639122"
+ y="13.543442"
+ id="tspan1129">It matches folders with name "ExperimentalData"</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="99.639122"
+ y="17.953154"
+ id="tspan1131">that are located in the crawler root folder.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="110.16309"
+ y="38.105129"
+ id="text1127-35"><tspan
+ sodipodi:role="line"
+ id="tspan1125-6"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="110.16309"
+ y="38.105129">Converter-Node on level 1:</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="110.16309"
+ y="42.514843"
+ id="tspan1131-9">It matches folders that have a name matching the given</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="110.16309"
+ y="46.924553"
+ id="tspan1286">regular expression (e.g. "2022_TestData"). All subfolders</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="110.16309"
+ y="51.334267"
+ id="tspan2018">in "ExperimentalData" are considered.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="87.131874"
+ y="58.891338"
+ id="text1127-35-1"><tspan
+ sodipodi:role="line"
+ id="tspan1125-6-2"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="87.131874"
+ y="58.891338">Create a "Project" record:</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="87.131874"
+ y="63.301052"
+ id="tspan2018-9">For each matching folder on level 1, a CaosDB record is created.</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="87.131874"
+ y="67.710762"
+ id="tspan2080">This record has one parent (name "Project") and two properties "date"</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="87.131874"
+ y="72.120476"
+ id="tspan2082">and "identifier". The two respective values are taken from the matched</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="87.131874"
+ y="76.53019"
+ id="tspan2084">regular expression. The dollar-signs indicate that the two variables,</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="87.131874"
+ y="80.939903"
+ id="tspan2086">which are created by the regular expression, are substituted.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="66.51297"
+ y="91.64978"
+ id="text1127-35-1-3"><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="66.51297"
+ y="91.64978"
+ id="tspan2086-1"><tspan
+ style="font-weight:bold"
+ id="tspan2175">Level 2 node "measurement"</tspan> uses a more complex regular expression</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="66.51297"
+ y="96.059494"
+ id="tspan2173">to match subfolders of the project folder.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="94.849007"
+ y="123.55322"
+ id="text1127-35-1-3-8"><tspan
+ sodipodi:role="line"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="94.849007"
+ y="123.55322"
+ id="tspan2173-2">Measurement record:</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.849007"
+ y="127.96294"
+ id="tspan2278">As no parents are given, the parent is automatically set to the name,</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.849007"
+ y="132.37265"
+ id="tspan2280">in this case "Measurement". Apart from the two properties stemming</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.849007"
+ y="136.78236"
+ id="tspan2282">from the regexp (date and identifier), a reference property is created</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.849007"
+ y="141.19208"
+ id="tspan2284">that creates a link to the Project record that was created earlier.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="94.862091"
+ y="162.61325"
+ id="text1127-35-1-3-8-2"><tspan
+ sodipodi:role="line"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="94.862091"
+ y="162.61325"
+ id="tspan2173-2-3">"dat"-Files on level 3:</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.862091"
+ y="167.02296"
+ id="tspan2284-2">Here, files are matched that end in ".dat".</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="94.639061"
+ y="185.28622"
+ id="text1127-35-1-3-8-2-2"><tspan
+ sodipodi:role="line"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="94.639061"
+ y="185.28622"
+ id="tspan2173-2-3-8">File records:</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.639061"
+ y="189.69594"
+ id="tspan2284-2-9">For each of the matched files create a file entity in CaosDB</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="94.639061"
+ y="194.10565"
+ id="tspan3189">and set the path accordingly.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="107.82945"
+ y="202.48438"
+ id="text1127-35-1-3-8-2-2-7"><tspan
+ sodipodi:role="line"
+ style="font-weight:bold;stroke-width:0.264583"
+ x="107.82945"
+ y="202.48438"
+ id="tspan2173-2-3-8-3">File properties:</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="107.82945"
+ y="206.89409"
+ id="tspan3189-1">The file is added as a reference property (called "output")</tspan><tspan
+ sodipodi:role="line"
+ style="font-weight:normal;stroke-width:0.264583"
+ x="107.82945"
+ y="211.3038"
+ id="tspan3238">to the Measurement record.</tspan></text><text
+ xml:space="preserve"
+ style="font-size:3.52777px;line-height:1.25;font-family:sans-serif;stroke-width:0.264583"
+ x="45.830696"
+ y="27.973518"
+ id="text1127-3"><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="45.830696"
+ y="27.973518"
+ id="tspan1131-5"><tspan
+ id="tspan1234"
+ style="font-weight:bold;stroke-width:0.264583">"subtree" </tspan>provides access to child-StructureElements,</tspan><tspan
+ sodipodi:role="line"
+ style="stroke-width:0.264583"
+ x="45.830696"
+ y="32.383232"
+ id="tspan1237">in this case the children of a directory converter are subdirectories.</tspan></text><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect1133"
+ width="32.761078"
+ height="4.9509659"
+ x="2.1495595"
+ y="27.815798" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:0.999996;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect2020"
+ width="105.07613"
+ height="13.460022"
+ x="2.0784345"
+ y="37.222599" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect2022"
+ width="82.460075"
+ height="26.69338"
+ x="1.8999104"
+ y="55.421516" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect2088"
+ width="185.95239"
+ height="15.581697"
+ x="1.5306897"
+ y="99.948242" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect2227"
+ width="90.042404"
+ height="18.340836"
+ x="1.019053"
+ y="122.36252" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:0.999996;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect2227-0"
+ width="88.036903"
+ height="14.435826"
+ x="1.6963811"
+ y="157.79872" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect3146"
+ width="88.402161"
+ height="17.550062"
+ x="1.8143678"
+ y="181.32809" /><rect
+ style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linejoin:round;stroke-dasharray:none"
+ id="rect3148"
+ width="102.97673"
+ height="5.1952801"
+ x="1.9082112"
+ y="203.22098" /></g></svg>
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
index 1652515968c3b0025a2916604632d57c042f119b..88d598ece284e1aad315a1e0fcae3fdf494b3aad 100644
--- a/src/doc/tutorials/index.rst
+++ b/src/doc/tutorials/index.rst
@@ -1,2 +1,9 @@
Tutorials
+++++++++
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+ Example CFood<example>
diff --git a/synchronize.md b/synchronize.md
index 30cf342cd4c4342ab43ed05799faf8b89abce71a..b178e647866d1e01c85ccfc8bff3383d5f93d21d 100644
--- a/synchronize.md
+++ b/synchronize.md
@@ -31,4 +31,3 @@ Maybe keep another dict that tracks what Record objects are in the to_be_updated
After treating leaf Records, Records that could not be checked before can be checked: Either referenced Records now have an ID or they are in the to_be_inserted dict such that it is clear that the identifiable at hand does not exist in the server.
This way, the whole structure can be resolved except if there are circular dependencies: Those can be added fully to the to_be_inserted dict. (???)
-
diff --git a/tox.ini b/tox.ini
index 101904b7de43fba6f04cf65641f555d79b0b080a..a7d4465ed36f0fe5e49c06721d3e3a0cdf453fa0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py38, py39, py310
+envlist = py37, py38, py39, py310, py311
skip_missing_interpreters = true
[testenv]
@@ -9,7 +9,12 @@ deps = .
# TODO: Make this f-branch sensitive
git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
-commands= caosdb-crawler --help
+commands = caosdb-crawler --help
py.test --cov=caosdb -vv {posargs}
+
[flake8]
-max-line-length=100
+max-line-length = 100
+
+[pytest]
+testpaths = unittests
+xfail_strict = True
\ No newline at end of file
diff --git a/unittests/broken_cfoods/broken1.yml b/unittests/broken_cfoods/broken1.yml
index 9fd4c52934c56512ada8ea564ccd540e07e25661..86202acd7a3be90b6a8b8e85aee5109d79799239 100644
--- a/unittests/broken_cfoods/broken1.yml
+++ b/unittests/broken_cfoods/broken1.yml
@@ -39,14 +39,14 @@ DataAnalysis: # name of the converter
# how to make match case insensitive?
subtree:
description:
- type: DictTextElement
+ type: TextElement
match_value: (?P<description>.*)
match_name: description
records:
Measurement:
description: $description
responsible_single:
- type: DictTextElement
+ type: TextElement
match_name: responsible
match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
records: &responsible_records
@@ -65,7 +65,7 @@ DataAnalysis: # name of the converter
subtree:
Person:
type: TextElement
- match: *person_regexp
+ match_name: *person_regexp
records: *responsible_records
ExperimentalData: # name of the converter
diff --git a/unittests/scifolder_cfood.yml b/unittests/scifolder_cfood.yml
index 90f193444bfda7296c46260236274da2378635cc..74fd027563907c5ae416ca389faba0ecd64d5848 100644
--- a/unittests/scifolder_cfood.yml
+++ b/unittests/scifolder_cfood.yml
@@ -42,14 +42,14 @@ Data: # name of the converter
# how to make match case insensitive?
subtree:
description:
- type: DictTextElement
+ type: TextElement
match_value: (?P<description>.*)
match_name: description
records:
Measurement:
description: $description
responsible_single:
- type: DictTextElement
+ type: TextElement
match_name: responsible
match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
records: &responsible_records
@@ -68,7 +68,7 @@ Data: # name of the converter
subtree:
Person:
type: TextElement
- match: *person_regexp
+ match_value: *person_regexp
records: *responsible_records
ExperimentalData: # name of the converter
diff --git a/unittests/scifolder_extended.yml b/unittests/scifolder_extended.yml
index 9bab612b9b37e8e295ee8fd02575de506a98d8fc..26f510679ff723ce5d9c0e705609e39bce60cbde 100644
--- a/unittests/scifolder_extended.yml
+++ b/unittests/scifolder_extended.yml
@@ -55,14 +55,14 @@ Data: # name of the converter
subtree:
description:
- type: DictTextElement
+ type: TextElement
match_value: (?P<description>.*)
match_name: description
records:
Measurement:
description: $description
responsible_single:
- type: DictTextElement
+ type: TextElement
match_name: responsible
match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
records: &responsible_records
@@ -76,12 +76,12 @@ Data: # name of the converter
# "responsible" belonging to Measurement.
responsible_list:
- type: DictListElement
+ type: ListElement
match_name: responsible
subtree:
Person:
type: TextElement
- match: *person_regexp
+ match_value: *person_regexp
records: *responsible_records
# sources_list:
diff --git a/unittests/scifolder_extended2.yml b/unittests/scifolder_extended2.yml
index 969325e91da488011819c338708a33dcfc32c93e..a189e79c12c2e1393188c8b9f532162518244508 100644
--- a/unittests/scifolder_extended2.yml
+++ b/unittests/scifolder_extended2.yml
@@ -56,14 +56,14 @@ Data: # name of the converter
subtree:
description:
- type: DictTextElement
+ type: TextElement
match_value: (?P<description>.*)
match_name: description
records:
Measurement:
description: $description
responsible_single:
- type: DictTextElement
+ type: TextElement
match_name: responsible
match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
records: &responsible_records
@@ -77,12 +77,12 @@ Data: # name of the converter
# "responsible" belonging to Measurement.
responsible_list:
- type: DictListElement
+ type: ListElement
match_name: responsible
subtree:
Person:
type: TextElement
- match: *person_regexp
+ match_value: *person_regexp
records: *responsible_records
# sources_list:
diff --git a/unittests/test_cache.py b/unittests/test_cache.py
deleted file mode 100644
index 135316b92fda0ac1e43f4e5f2c4f28fbf1272494..0000000000000000000000000000000000000000
--- a/unittests/test_cache.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/python
-# Tests for entity comparison
-# A. Schlemmer, 06/2021
-
-import caosdb as db
-from pytest import raises
-
-from caoscrawler.identified_cache import _create_hashable_string as create_hash_string
-
-
-def test_normal_hash_creation():
- # Test the initial functionality:
- # hash comprises only one parent, name and properties:
-
- r1 = db.Record()
- r1.add_property(name="test")
- r1.add_parent("bla")
- hash1 = create_hash_string(r1)
-
- r2 = db.Record()
- r2.add_property(name="test2")
- r2.add_parent("bla")
- hash2 = create_hash_string(r2)
-
- assert hash1 != hash2
-
- r3 = db.Record()
- r3.add_property(name="test")
- r3.add_parent("bla bla")
- hash3 = create_hash_string(r3)
- assert hash1 != hash3
- assert hash2 != hash3
-
- # no name and no properties and no parents:
- r4 = db.Record()
- with raises(RuntimeError, match=".*1 parent.*"):
- create_hash_string(r4)
-
- # should work
- r4.add_parent("bla")
- assert len(create_hash_string(r4)) > 0
- r4.add_property(name="test")
- assert len(create_hash_string(r4)) > 0
-
- r4.add_parent("bla bla")
- with raises(RuntimeError, match=".*1 parent.*"):
- create_hash_string(r4)
-
-
-def test_file_hash_creation():
- f1 = db.File(path="/bla/bla/test1.txt")
- hash1 = create_hash_string(f1)
- f2 = db.File(path="/bla/bla/test2.txt")
- hash2 = create_hash_string(f2)
-
- assert hash1 != hash2
diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..09d6c88bdc27e1066ed18a9c5865cbfb95270c3a
--- /dev/null
+++ b/unittests/test_cfood_metadata.py
@@ -0,0 +1,199 @@
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import pytest
+import yaml
+
+from tempfile import NamedTemporaryFile
+
+import caoscrawler
+
+CRAWLER_VERSION = ""
+
+
+def setup_function(function):
+ """Store original crawler version in case it is altered for tests."""
+ CRAWLER_VERSION = caoscrawler.version.version
+
+
+def teardown_function(function):
+ """Reset version"""
+ caoscrawler.version.version = CRAWLER_VERSION
+
+
+def _temp_file_load(txt: str):
+ """
+ Create a temporary file with txt and load the crawler
+ definition using load_definition from Crawler.
+ """
+ definition = None
+ with NamedTemporaryFile() as f:
+ f.write(txt.encode())
+ f.flush()
+ c = caoscrawler.Crawler()
+ definition = c.load_definition(f.name)
+ return definition
+
+
+def test_warning_if_no_version_specified():
+ """Warn if no version is specified in the cfood."""
+
+ # metadata section exists but doesn't specify a version
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+ assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+ # metadata section is missing alltogether
+ definition_text = """
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+ assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+
+def test_warning_if_version_too_old():
+ """Warn if the cfood was written for an older crawler version."""
+
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.0
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+
+ # higher minor
+ caoscrawler.version.version = "0.3.0"
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+ assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+ assert "version installed on your system: 0.3.0" in uw[0].message.args[0]
+
+ # higher major
+ caoscrawler.version.version = "1.1.0"
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+ assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+ assert "version installed on your system: 1.1.0" in uw[0].message.args[0]
+
+
+def test_error_if_version_too_new():
+ """Raise error if the cfood requires a newer crawler version."""
+
+ # minor too old
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ caoscrawler.version.version = "0.1.5"
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 0.2.1" in str(cre.value)
+ assert "version installed on your system: 0.1.5" in str(cre.value)
+
+ # major too old
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 1.0.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 1.0.1" in str(cre.value)
+ assert "version installed on your system: 0.1.5" in str(cre.value)
+
+ # patch to old
+ caoscrawler.version.version = "1.0.0"
+
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 1.0.1" in str(cre.value)
+ assert "version installed on your system: 1.0.0" in str(cre.value)
+
+
+def test_matching_version():
+ """Test that there is no warning or error in case the version matches."""
+
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ caoscrawler.version.version = "0.2.1"
+ assert _temp_file_load(definition_text)
+
+ # The version is also considered a match if the patch version of the
+ # installed crawler is newer than the one specified in the cfood metadata
+ caoscrawler.version.version = "0.2.7"
+ assert _temp_file_load(definition_text)
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 8562ce51132466b293bcc2d2aea5cb05f2985418..5942b1e124ebd1228a619ed7a1024738c70ee0aa 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -23,23 +23,25 @@
"""
test the converters module
"""
+import json
+import yaml
import importlib
import os
+from itertools import product
import pytest
import yaml
-from caoscrawler.converters import (Converter, ConverterValidationError,
- DictConverter, DirectoryConverter, DictIntegerElementConverter,
+from caoscrawler.converters import (Converter, ConverterValidationError, DictElementConverter,
+ DirectoryConverter, DictIntegerElementConverter,
handle_value, MarkdownFileConverter,
- DictFloatElementConverter, JSONFileConverter)
-from caoscrawler.converters import _AbstractDictElementConverter
+ FloatElementConverter, IntegerElementConverter,
+ JSONFileConverter, YAMLFileConverter)
+from caoscrawler.converters import _AbstractScalarValueElementConverter
from caoscrawler.crawl import Crawler
from caoscrawler.stores import GeneralStore
-from caoscrawler.structure_elements import (File, DictTextElement,
- DictListElement, DictElement,
- DictBooleanElement, DictDictElement,
- DictIntegerElement,
- DictFloatElement, Directory)
+from caoscrawler.structure_elements import (File, TextElement, ListElement, DictElement,
+ BooleanElement, IntegerElement,
+ FloatElement, Directory)
from test_tool import rfp
@@ -53,14 +55,14 @@ def converter_registry():
"MarkdownFile": {
"converter": "MarkdownFileConverter",
"package": "caoscrawler.converters"},
- "Dict": {
- "converter": "DictConverter",
+ "DictElement": {
+ "converter": "DictElementConverter",
"package": "caoscrawler.converters"},
- "DictTextElement": {
- "converter": "DictTextElementConverter",
+ "TextElement": {
+ "converter": "TextElementConverter",
"package": "caoscrawler.converters"},
- "DictListElement": {
- "converter": "DictListElementConverter",
+ "ListElement": {
+ "converter": "ListElementConverter",
"package": "caoscrawler.converters"},
"TextElement": {
"converter": "TextElementConverter",
@@ -81,8 +83,8 @@ def testConverterTrivial(converter_registry):
types = [
"Directory",
"MarkdownFile",
- "DictTextElement",
- "DictListElement",
+ "TextElement",
+ "ListElement",
"TextElement"
]
@@ -150,11 +152,11 @@ def test_markdown_converter(converter_registry):
children = converter.create_children(None, test_readme)
assert len(children) == 5
- assert children[1].__class__ == DictTextElement
+ assert children[1].__class__ == TextElement
assert children[1].name == "description"
assert children[1].value.__class__ == str
- assert children[0].__class__ == DictTextElement
+ assert children[0].__class__ == TextElement
assert children[0].name == "responsible"
assert children[0].value.__class__ == str
@@ -170,11 +172,11 @@ def test_markdown_converter(converter_registry):
children = converter.create_children(None, test_readme2)
assert len(children) == 2
- assert children[1].__class__ == DictTextElement
+ assert children[1].__class__ == TextElement
assert children[1].name == "description"
assert children[1].value.__class__ == str
- assert children[0].__class__ == DictListElement
+ assert children[0].__class__ == ListElement
assert children[0].name == "responsible"
assert children[0].value.__class__ == list
@@ -194,55 +196,138 @@ def test_json_converter(converter_registry):
assert m is not None
assert len(m) == 0
- children = jsonconverter.create_children(None, test_json)
- assert len(children) == 8
- assert children[0].__class__ == DictTextElement
- assert children[0].name == "name"
- assert children[0].value.__class__ == str
- assert children[0].value == "DEMO"
+ dict_el = jsonconverter.create_children(None, test_json)
+ assert len(dict_el) == 1
- assert children[1].__class__ == DictIntegerElement
- assert children[1].name == "projectId"
- assert children[1].value.__class__ == int
- assert children[1].value == 10002
+ dictconverter = DictElementConverter(
+ definition={"match_name": "(.*)"},
+ name="dictconv",
+ converter_registry=converter_registry)
+ children = dictconverter.create_children(None, dict_el[0])
+ for child in children:
+ if child.name == "name":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ assert child.value == "DEMO"
+ elif child.name == "projectId":
+ assert isinstance(child, IntegerElement)
+ assert isinstance(child.value, int)
+ assert child.value == 10002
+ elif child.name == "archived":
+ assert isinstance(child, BooleanElement)
+ assert isinstance(child.value, bool)
+ assert child.value is False
+ elif child.name == "Person":
+ assert isinstance(child, ListElement)
+ assert isinstance(child.value, list)
+ assert len(child.value) == 2
+ elif child.name == "start_date":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ assert child.value == '2022-03-01'
+ elif child.name == "candidates":
+ assert isinstance(child, ListElement)
+ assert isinstance(child.value, list)
+ assert child.value == ["Mouse", "Penguine"]
+ elif child.name == "rvalue":
+ assert isinstance(child, FloatElement)
+ assert isinstance(child.value, float)
+ elif child.name == "url":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ else:
+ raise ValueError()
+
+ invalid_json = File(
+ "invalidjson.json",
+ rfp("test_directories", "examples_json", "invalidjson.json")
+ )
+ # Doesn't validate because of missing required 'name' property
+ with pytest.raises(ConverterValidationError) as err:
+ jsonconverter.create_children(None, invalid_json)
+ assert err.value.message.startswith("Couldn't validate")
- assert children[2].__class__ == DictBooleanElement
- assert children[2].name == "archived"
- assert children[2].value.__class__ == bool
+ broken_json = File(
+ "brokenjson.json",
+ rfp("test_directories", "examples_json", "brokenjson.json")
+ )
+ with pytest.raises(json.decoder.JSONDecodeError) as err:
+ jsonconverter.create_children(None, broken_json)
- assert children[3].__class__ == DictListElement
- assert children[3].name == "Person"
- assert children[3].value.__class__ == list
- assert len(children[3].value) == 2
- assert children[4].__class__ == DictTextElement
- assert children[4].name == "start_date"
- assert children[4].value.__class__ == str
+def test_yaml_converter(converter_registry):
+ test_yaml = File("testyaml.yml", rfp(
+ "test_directories", "test_yamls", "testyaml.yml"))
- assert children[5].__class__ == DictListElement
- assert children[5].name == "candidates"
- assert children[5].value.__class__ == list
- assert children[5].value == ["Mouse", "Penguine"]
+ schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ "test_directories", "test_yamls", "testyaml.schema.json")
+ yamlconverter = YAMLFileConverter(
+ definition={"match": "(.*)", "validate": schema_path},
+ name="TestYAMLFileConverter",
+ converter_registry=converter_registry)
- assert children[6].__class__ == DictFloatElement
- assert children[6].name == "rvalue"
- assert children[6].value.__class__ == float
+ m = yamlconverter.match(test_yaml)
+ assert m is not None
+ assert len(m) == 0
- assert children[7].__class__ == DictTextElement
- assert children[7].name == "url"
- assert children[7].value.__class__ == str
+ dict_el = yamlconverter.create_children(None, test_yaml)
+ assert len(dict_el) == 1
- broken_json = File(
- "brokenjson.json",
- rfp("test_directories", "examples_json", "brokenjson.json")
+ dictconverter = DictElementConverter(
+ definition={"match_name": "(.*)"},
+ name="dictconv",
+ converter_registry=converter_registry)
+ children = dictconverter.create_children(None, dict_el[0])
+ for child in children:
+ if child.name == "name":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ assert child.value == "DEMO"
+ elif child.name == "projectId":
+ assert isinstance(child, IntegerElement)
+ assert isinstance(child.value, int)
+ assert child.value == 10002
+ elif child.name == "archived":
+ assert isinstance(child, BooleanElement)
+ assert isinstance(child.value, bool)
+ assert child.value is False
+ elif child.name == "Person":
+ assert isinstance(child, ListElement)
+ assert isinstance(child.value, list)
+ assert len(child.value) == 2
+ elif child.name == "start_date":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ assert child.value == '2022-03-01'
+ elif child.name == "candidates":
+ assert isinstance(child, ListElement)
+ assert isinstance(child.value, list)
+ assert child.value == ["Mouse", "Penguine"]
+ elif child.name == "rvalue":
+ assert isinstance(child, FloatElement)
+ assert isinstance(child.value, float)
+ elif child.name == "url":
+ assert isinstance(child, TextElement)
+ assert isinstance(child.value, str)
+ else:
+ raise ValueError()
+
+ invalid_yaml = File(
+ "invalidyaml.yml",
+ rfp("test_directories", "test_yamls", "invalidyaml.yml")
)
- m = jsonconverter.match(broken_json)
# Doesn't validate because of missing required 'name' property
with pytest.raises(ConverterValidationError) as err:
- children = jsonconverter.create_children(None, broken_json)
+ yamlconverter.create_children(None, invalid_yaml)
+ assert err.value.message.startswith("Couldn't validate")
- assert err.value.message.startswith("Couldn't validate")
+ broken_yaml = File(
+ "brokenyaml.yml",
+ rfp("test_directories", "test_yamls", "brokenyaml.yml")
+ )
+ with pytest.raises(yaml.parser.ParserError) as err:
+ yamlconverter.create_children(None, broken_yaml)
def test_variable_replacement():
@@ -274,7 +359,7 @@ def test_variable_replacement():
assert handle_value(["$a", "$b"], values) == (["4", "68"], "single")
-def test_filter_children_of_directory(converter_registry):
+def test_filter_children_of_directory(converter_registry, capsys):
"""Verify that children (i.e., files) in a directory are filtered or sorted
correctly.
@@ -285,6 +370,7 @@ def test_filter_children_of_directory(converter_registry):
dc = DirectoryConverter(
definition={
"match": "(.*)",
+ "debug_match": True,
"filter": {
"expr": "test_(?P<date>[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}).json",
"group": "date",
@@ -297,6 +383,14 @@ def test_filter_children_of_directory(converter_registry):
m = dc.match(test_dir)
assert m is not None
+ # checking debug output
+ captured = capsys.readouterr()
+ # the name
+ assert "examples_filter_children" in captured.out
+ # the regexp
+ assert "(.*)" in captured.out
+ # the empty result set
+ assert "{}" in captured.out
# This should only contain the youngest json and the csv that doesn't match
# the above filter expression.
@@ -394,11 +488,11 @@ match_name: text
match_value: .*begin(?P<text>.*)end
accept_text: True
""")
- converter = _AbstractDictElementConverter(
+ converter = _AbstractScalarValueElementConverter(
definition, "test_converter",
None # This is possible when "subtree" is not used
)
- element = DictTextElement("text", """
+ element = TextElement("text", """
begin
bla
end""")
@@ -409,7 +503,7 @@ end""")
def test_converter_value_match(converter_registry):
# test with defaults
- dc = DictFloatElementConverter(
+ dc = FloatElementConverter(
definition={
"match_name": "(.*)",
"match_value": "(.*)",
@@ -417,11 +511,11 @@ def test_converter_value_match(converter_registry):
name="Test",
converter_registry=converter_registry
)
- m = dc.match(DictIntegerElement(name="a", value=4))
+ m = dc.match(IntegerElement(name="a", value=4))
assert m is not None
# overwrite default with no match for int
- dc = DictFloatElementConverter(
+ dc = FloatElementConverter(
definition={
"match_name": "(.*)",
"match_value": "(.*)",
@@ -430,11 +524,10 @@ def test_converter_value_match(converter_registry):
name="Test",
converter_registry=converter_registry
)
- with pytest.raises(RuntimeError) as err:
- m = dc.match(DictIntegerElement(name="a", value=4))
+ assert dc.typecheck(IntegerElement(name="a", value=4)) is False
# overwrite default with match for float
- dc = DictIntegerElementConverter(
+ dc = IntegerElementConverter(
definition={
"match_name": "(.*)",
"match_value": "(.*)",
@@ -443,5 +536,37 @@ def test_converter_value_match(converter_registry):
name="Test",
converter_registry=converter_registry
)
- m = dc.match(DictFloatElement(name="a", value=4.0))
+ m = dc.match(FloatElement(name="a", value=4.0))
assert m is not None
+
+
+def test_match_debug(converter_registry, capsys):
+ for m, mn, mv in product([".*", None], [".*", None], [".*", None]):
+ defi = {"debug_match": True}
+ if m:
+ defi["match"] = m
+ if mn:
+ defi["match_name"] = mn
+ if mv:
+ defi["match_value"] = mv
+ dc = FloatElementConverter(
+ definition=defi,
+ name="Test",
+ converter_registry=converter_registry
+ )
+ if m and mn:
+ with pytest.raises(RuntimeError) as err:
+ mtch = dc.match(IntegerElement(name="a", value=4))
+ continue
+ else:
+ mtch = dc.match(IntegerElement(name="a", value=4))
+ if not (m is None and mn is None and mv is None):
+ assert mtch is not None
+ # checking debug output
+ captured = capsys.readouterr()
+ # the name
+ assert "a" in captured.out
+ # the regexp
+ assert ".*" in captured.out
+ # the empty result set
+ assert "{}" in captured.out
diff --git a/unittests/test_data/failing_validation/cfood.yml b/unittests/test_data/failing_validation/cfood.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e0bc8668457cefaedd5244c424c99a150e30347c
--- /dev/null
+++ b/unittests/test_data/failing_validation/cfood.yml
@@ -0,0 +1,11 @@
+# This is a test cfood for:
+# https://gitlab.com/caosdb/caosdb-crawler/-/issues/9
+
+Data: # name of the converter
+ type: Directory
+ match: (.*)
+ subtree:
+ json:
+ type: JSONFile
+ match: data.json
+ validate: schema.json
diff --git a/unittests/test_data/failing_validation/cfood2.yml b/unittests/test_data/failing_validation/cfood2.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d896553f7acbdf4fe149bd8bd0e63c96bc121916
--- /dev/null
+++ b/unittests/test_data/failing_validation/cfood2.yml
@@ -0,0 +1,14 @@
+# This is a test cfood for:
+# https://gitlab.com/caosdb/caosdb-crawler/-/issues/9
+
+Data: # name of the converter
+ type: Directory
+ match: (.*)
+ subtree:
+ json:
+ type: JSONFile
+ match: data.json
+ subtree:
+ dict:
+ type: Dict
+ validate: schema.json
diff --git a/unittests/test_data/failing_validation/data.json b/unittests/test_data/failing_validation/data.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6ecf65f31a974233d8bf5f1b779e0718ea41258
--- /dev/null
+++ b/unittests/test_data/failing_validation/data.json
@@ -0,0 +1,3 @@
+{
+ "a": 5
+}
diff --git a/unittests/test_data/failing_validation/identifiables.yml b/unittests/test_data/failing_validation/identifiables.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f6bb04a59bda2e6169e3dc037a60686f6fd935a3
--- /dev/null
+++ b/unittests/test_data/failing_validation/identifiables.yml
@@ -0,0 +1,3 @@
+
+license:
+ - name
diff --git a/unittests/test_data/failing_validation/schema.json b/unittests/test_data/failing_validation/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..657d89842c3941c68da7bfeff7f52026d26b5f6f
--- /dev/null
+++ b/unittests/test_data/failing_validation/schema.json
@@ -0,0 +1,10 @@
+{
+ "title": "Dataset",
+ "description": "",
+ "type": "object",
+ "properties": {
+ "a": {
+ "type": "string"
+ }
+ }
+}
diff --git a/unittests/test_directories/examples_json/brokenjson.json b/unittests/test_directories/examples_json/brokenjson.json
index 9c012bf062264014278fc2df7be6cf33b65c7469..20b17a7e6767c3f9f2569d5b9d5711940845857a 100644
--- a/unittests/test_directories/examples_json/brokenjson.json
+++ b/unittests/test_directories/examples_json/brokenjson.json
@@ -1,13 +1 @@
-{
- "projectId": 10002,
- "archived": false,
- "coordinator": {
- "firstname": "Miri",
- "lastname": "Mueller",
- "email": "miri.mueller@science.de"
- },
- "start_date": "2022-03-01",
- "candidates": ["Mouse", "Penguine"],
- "rvalue": 0.4444,
- "url": "https://site.de/index.php/"
-}
+a: 5
diff --git a/unittests/test_directories/examples_json/invalidjson.json b/unittests/test_directories/examples_json/invalidjson.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c012bf062264014278fc2df7be6cf33b65c7469
--- /dev/null
+++ b/unittests/test_directories/examples_json/invalidjson.json
@@ -0,0 +1,13 @@
+{
+ "projectId": 10002,
+ "archived": false,
+ "coordinator": {
+ "firstname": "Miri",
+ "lastname": "Mueller",
+ "email": "miri.mueller@science.de"
+ },
+ "start_date": "2022-03-01",
+ "candidates": ["Mouse", "Penguine"],
+ "rvalue": 0.4444,
+ "url": "https://site.de/index.php/"
+}
diff --git a/unittests/test_directories/examples_json/jsontest_cfood.yml b/unittests/test_directories/examples_json/jsontest_cfood.yml
index f1eb6a9fa186c07f551bd12a84050f544abfdabc..2dcef5c7da6d2bee18ae7807210ad297915ea397 100644
--- a/unittests/test_directories/examples_json/jsontest_cfood.yml
+++ b/unittests/test_directories/examples_json/jsontest_cfood.yml
@@ -8,51 +8,55 @@ JSONTest: # name of the converter
parents:
- Project # not needed as the name is equivalent
subtree:
- name_element:
- type: DictTextElement
- match_name: "name"
- match_value: "(?P<name>.*)"
- records:
- Project:
- name: $name
- url_element: # name of the first subtree element which is a converter
- type: DictTextElement
- match_value: "(?P<url>.*)"
- match_name: "url"
- records:
- Project:
- url: $url
- persons_element:
- type: DictListElement
- match_name: "Person"
+ dictel:
+ type: DictElement
+ match: '(.*)'
subtree:
- person_element:
- type: Dict
+ name_element:
+ type: TextElement
+ match_name: "name"
+ match_value: "(?P<name>.*)"
records:
- Person:
- parents:
- - Person
Project:
- Person: +$Person
+ name: $name
+ url_element: # name of the first subtree element which is a converter
+ type: TextElement
+ match_value: "(?P<url>.*)"
+ match_name: "url"
+ records:
+ Project:
+ url: $url
+ persons_element:
+ type: ListElement
+ match_name: "Person"
subtree:
- firstname_element:
- type: DictTextElement
- match_name: "firstname"
- match_value: "(?P<firstname>.*)"
- records:
- Person:
- firstname: $firstname
- lastname_element:
- type: DictTextElement
- match_name: "lastname"
- match_value: "(?P<lastname>.*)"
- records:
- Person:
- lastname: $lastname
- email_element:
- type: DictTextElement
- match_name: "email"
- match_value: "(?P<email>.*)"
+ person_element:
+ type: DictElement
records:
Person:
- email: $email
+ parents:
+ - Person
+ Project:
+ Person: +$Person
+ subtree:
+ firstname_element:
+ type: TextElement
+ match_name: "firstname"
+ match_value: "(?P<firstname>.*)"
+ records:
+ Person:
+ firstname: $firstname
+ lastname_element:
+ type: TextElement
+ match_name: "lastname"
+ match_value: "(?P<lastname>.*)"
+ records:
+ Person:
+ lastname: $lastname
+ email_element:
+ type: TextElement
+ match_name: "email"
+ match_value: "(?P<email>.*)"
+ records:
+ Person:
+ email: $email
diff --git a/unittests/test_directories/single_file_test_data/identifiables.yml b/unittests/test_directories/single_file_test_data/identifiables.yml
index e32746d5a6984096cc46fa618250832b325965b0..c6f82be3dbf11db3f69e06d9a6fd2ee692901212 100644
--- a/unittests/test_directories/single_file_test_data/identifiables.yml
+++ b/unittests/test_directories/single_file_test_data/identifiables.yml
@@ -5,3 +5,7 @@ Keyword:
Project:
- project_id
- title
+Unknown:
+ - propa
+ - is_referenced_by: [Some]
+
diff --git a/unittests/test_directories/test_yamls/brokenyaml.yml b/unittests/test_directories/test_yamls/brokenyaml.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5c34318c2147fb2285de5a1513f46168e33e6baf
--- /dev/null
+++ b/unittests/test_directories/test_yamls/brokenyaml.yml
@@ -0,0 +1 @@
+}
diff --git a/unittests/test_directories/test_yamls/invalidyaml.yml b/unittests/test_directories/test_yamls/invalidyaml.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0528ebea1740947f7e675cf77f906fd86beaa6f9
--- /dev/null
+++ b/unittests/test_directories/test_yamls/invalidyaml.yml
@@ -0,0 +1,14 @@
+"projectId": 10002
+"archived": false
+"Person":
+ - "firstname": "Miri"
+ "lastname": "Mueller"
+ "other": null
+ "email": "miri.mueller@science.de"
+ - "firstname": "Mara"
+ "lastname": "Mueller"
+ "email": "mara.mueller@science.de"
+"start_date": "2022-03-01"
+"candidates": ["Mouse", "Penguine"]
+"rvalue": 0.4444
+"url": "https://site.de/index.php/"
diff --git a/unittests/test_directories/test_yamls/testyaml.schema.json b/unittests/test_directories/test_yamls/testyaml.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc784a61079e4737f1a0176fe4240133f5d1b5d0
--- /dev/null
+++ b/unittests/test_directories/test_yamls/testyaml.schema.json
@@ -0,0 +1,60 @@
+{
+ "title": "Dataset",
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "projectId": {
+ "type": "integer"
+ },
+ "archived": {
+ "type": "boolean"
+ },
+ "Person": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "firstname": {
+ "type": "string"
+ },
+ "lastname": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "firstname",
+ "lastname",
+ "email"
+ ],
+ "additionalProperties": true
+ }
+ },
+ "start_date": {
+ "type": "string",
+ "format": "date"
+ },
+ "candidates": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "rvalue": {
+ "type": "number"
+ },
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "name",
+ "projectId",
+ "Person"
+ ],
+ "additionalProperties": false
+}
diff --git a/unittests/test_directories/test_yamls/testyaml.yml b/unittests/test_directories/test_yamls/testyaml.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d658e187f62b1b66521be385ae34386273b3b98a
--- /dev/null
+++ b/unittests/test_directories/test_yamls/testyaml.yml
@@ -0,0 +1,15 @@
+"name": "DEMO"
+"projectId": 10002
+"archived": false
+"Person":
+ - "firstname": "Miri"
+ "lastname": "Mueller"
+ "other": null
+ "email": "miri.mueller@science.de"
+ - "firstname": "Mara"
+ "lastname": "Mueller"
+ "email": "mara.mueller@science.de"
+"start_date": "2022-03-01"
+"candidates": ["Mouse", "Penguine"]
+"rvalue": 0.4444
+"url": "https://site.de/index.php/"
diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py
index b0b9801993dc68fe473e788b8ca79a2244912676..aff174d0228d2750efd1cca129547c821c974127 100644
--- a/unittests/test_file_identifiables.py
+++ b/unittests/test_file_identifiables.py
@@ -8,63 +8,44 @@ import pytest
from pytest import raises
from caoscrawler.identifiable_adapters import LocalStorageIdentifiableAdapter
+from caoscrawler.identifiable import Identifiable
def test_file_identifiable():
ident = LocalStorageIdentifiableAdapter()
- file_obj = db.File()
+ # Without a path there is no identifying information
+ with raises(ValueError):
+ ident.get_identifiable(db.File(), [])
+
+ fp = "/test/bla/bla.txt"
+ file_obj = db.File(path=fp)
identifiable = ident.get_identifiable(file_obj)
- identifiable2 = ident.get_identifiable_for_file(file_obj)
- # these are two different objects:
- assert identifiable != identifiable2
- assert file_obj != identifiable
- # ... but the path is equal:
- assert identifiable.path == identifiable2.path
- # ... and very boring:
- assert identifiable.path is None
- # Test functionality of retrieving the files:
- identified_file = ident.get_file(identifiable)
- identified_file2 = ident.get_file(file_obj)
- # The both should be None currently as there are no files in the local store yet:
- assert identified_file is None
- assert identified_file2 is None
+ # the path is copied to the identifiable
+ assert fp == identifiable.path
+ assert isinstance(identifiable, Identifiable)
- # Let's make it more interesting:
- file_obj.path = "/test/bla/bla.txt"
- file_obj._checksum = "abcd"
- identifiable = ident.get_identifiable(file_obj)
- assert file_obj != identifiable
- assert file_obj.path == identifiable.path
- # Checksum is not part of the identifiable:
- assert file_obj.checksum != identifiable.checksum
+ # __eq__ function is only defined for Identifiable objects
+ with raises(ValueError):
+ file_obj != identifiable
- # This is the wrong method, so it should definitely return None:
- identified_file = ident.retrieve_identified_record_for_identifiable(
- identifiable)
- assert identified_file is None
- # This is the correct method to use:
- identified_file = ident.get_file(identifiable)
- # or directly using:
- identified_file2 = ident.get_file(file_obj)
- # The both should be None currently as there are no files in the local store yet:
- assert identified_file is None
- assert identified_file2 is None
+ # since the path does not exist in the data in ident, the follwoing functions return None
+ assert ident.retrieve_identified_record_for_record(file_obj) is None
+ assert ident.get_file(identifiable) is None
# Try again with actual files in the store:
records = ident.get_records()
- test_record_wrong_path = db.File(
- path="/bla/bla/test.txt")
- test_record_correct_path = db.File(
- path="/test/bla/bla.txt")
- test_record_alsocorrect_path = db.File(
- path="/test/bla/bla.txt")
+ test_record_wrong_path = db.File(path="/bla/bla/test.txt")
+ test_record_correct_path = db.File(path="/test/bla/bla.txt")
+ test_record_alsocorrect_path = db.File(path="/test/bla/bla.txt")
records.append(test_record_wrong_path)
+ # Now, there is a file, but still wrong path -> result is still None
identified_file = ident.get_file(file_obj)
assert identified_file is None
records.append(test_record_correct_path)
+ # now there is a match
identified_file = ident.get_file(file_obj)
assert identified_file is not None
assert identified_file.path == file_obj.path
diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f3c606b163df4dc238be9a669fd31eb630a582d
--- /dev/null
+++ b/unittests/test_identifiable.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+test identifiable module
+"""
+
+import pytest
+import caosdb as db
+from caoscrawler.identifiable import Identifiable
+from caoscrawler.identified_cache import IdentifiedCache
+
+
+def test_create_hashable_string():
+ assert Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B")) == "P<B>N<A>R<[]>"
+ assert Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': 5})) == "P<B>N<A>R<[]>a:5"
+ a = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': 4, 'b': 5}))
+ b = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'b': 5, 'a': 4}))
+ assert a == b
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B",
+ properties={'a': db.Record(id=12)})
+ ) == "P<B>N<A>R<[]>a:12")
+ a = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': [db.Record(id=12)]}))
+ assert (a == "P<B>N<A>R<[]>a:[12]")
+ assert (Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': [12]})) == "P<B>N<A>R<[]>a:[12]")
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={
+ 'a': [db.Record(id=12), 11]})
+ ) == "P<B>N<A>R<[]>a:[12, 11]")
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(record_type="B", properties={'a': [db.Record()]})
+ ) != Identifiable._create_hashable_string(
+ Identifiable(record_type="B", properties={'a': [db.Record()]})))
+ assert Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", backrefs=[123, db.Entity(id=124)],
+ properties={'a': 5})) == "P<B>N<A>R<['123', '124']>a:5"
+
+
+def test_name():
+ with pytest.raises(ValueError):
+ Identifiable(properties={"Name": 'li'})
+
+
+def test_repr():
+ # only test that something meaningful is returned
+ assert 'properties' in str(Identifiable(name="A", record_type="B"))
+ assert str(Identifiable(name="A", record_type="B", properties={'a': 0})).split(
+ "properties:\n")[1].split('\n')[0] == '{"a": 0}'
+ assert str(Identifiable(name="A", record_type="B", properties={'a': 0, 'b': "test"})).split(
+ "properties:\n")[1].split('\n')[0] == '{"a": 0, "b": "test"}'
+
+ # TODO(henrik): Add a test using backrefs once that's implemented.
+
+
+def test_equality():
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) == Identifiable(record_id=12, properties={"a": 1})
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) != Identifiable(record_id=13, properties={"a": 0})
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) == Identifiable(properties={"a": 0})
+ assert Identifiable(
+ path="a", properties={"a": 0}) != Identifiable(path="b", properties={"a": 0})
+ assert Identifiable(
+ path="a", properties={"a": 0}) == Identifiable(path="a", properties={"a": 1})
+ assert Identifiable(
+ path="a", properties={"a": 0}) == Identifiable(properties={"a": 0})
+ assert Identifiable(properties={"a": 0}) == Identifiable(
+ properties={"a": 0})
+ assert Identifiable(properties={"a": 0}) != Identifiable(
+ properties={"a": 1})
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
index ef7998a460c07342d30a3f769fd609c1045a9cca..c3b44e1c8e3775fc6e8b7118b82ffa6a20bef484 100644
--- a/unittests/test_identifiable_adapters.py
+++ b/unittests/test_identifiable_adapters.py
@@ -30,36 +30,52 @@ test identifiable_adapters module
import os
from datetime import datetime
from caoscrawler.identifiable_adapters import (
- CaosDBIdentifiableAdapter, IdentifiableAdapter)
+ CaosDBIdentifiableAdapter, convert_value, IdentifiableAdapter)
+from caoscrawler.identifiable import Identifiable
import caosdb as db
def test_create_query_for_identifiable():
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record().add_parent("Person")
- .add_property("first_name", value="A")
- .add_property("last_name", value="B"))
+ Identifiable(record_type="Person", properties={"first_name": "A", "last_name": "B"}))
assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record(name="A").add_parent("B")
- .add_property("c", value="c")
- .add_property("d", value=5)
- .add_property("e", value=5.5)
- .add_property("f", value=datetime(2020, 10, 10))
- .add_property("g", value=True)
- .add_property("h", value=db.Record(id=1111))
- .add_property("i", value=db.File(id=1112))
- .add_property("j", value=[2222, db.Record(id=3333)]))
- assert (query.lower() == "find record b with name='a' and 'c'='c' and 'd'='5' and 'e'='5.5'"
- " and 'f'='2020-10-10t00:00:00' and 'g'='true' and 'h'='1111' and 'i'='1112' and "
- "'j'='2222' and 'j'='3333' ")
+ Identifiable(name="A", record_type="B", properties={
+ "c": "c",
+ "d": 5,
+ "e": 5.5,
+ "f": datetime(2020, 10, 10),
+ "g": True,
+ "h": db.Record(id=1111),
+ "i": db.File(id=1112),
+ "j": [2222, db.Record(id=3333)]}))
+ assert (query == "FIND RECORD B WITH name='A' AND 'c'='c' AND 'd'='5' AND 'e'='5.5'"
+ " AND 'f'='2020-10-10T00:00:00' AND 'g'='TRUE' AND 'h'='1111' AND 'i'='1112' AND "
+ "'j'='2222' AND 'j'='3333' ")
# The name can be the only identifiable
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record(name="TestRecord").add_parent("TestType"))
+ Identifiable(name="TestRecord", record_type="TestType"))
assert query.lower() == "find record testtype with name='testrecord'"
+ # With referencing entity (backref)
+ query = IdentifiableAdapter.create_query_for_identifiable(
+ Identifiable(record_type="Person", backrefs=[14433], properties={'last_name': "B"}))
+ assert query.lower() == ("find record person which is referenced by 14433 and with "
+ "'last_name'='b' ")
+
+ # With two referencing entities (backref)
+ query = IdentifiableAdapter.create_query_for_identifiable(
+ Identifiable(record_type="Person", backrefs=[14433, 333], properties={'last_name': "B"}))
+ assert query.lower() == ("find record person which is referenced by 14433 and which is "
+ "referenced by 333 and with 'last_name'='b' ")
+
+ # With single quote in string
+ query = IdentifiableAdapter.create_query_for_identifiable(
+ Identifiable(record_type="Person", backrefs=[], properties={'last_name': "B'Or"}))
+ assert query == ("FIND RECORD Person WITH 'last_name'='B\\'Or' ")
+
def test_load_from_yaml_file():
ident = CaosDBIdentifiableAdapter()
@@ -83,3 +99,12 @@ def test_load_from_yaml_file():
assert project_i is not None
assert project_i.get_property("project_id") is not None
assert project_i.get_property("title") is not None
+
+
+def test_convert_value():
+ # test that string representation of objects stay unchanged. No stripping or so.
+ class A():
+ def __repr__(self):
+ return " a "
+
+ assert convert_value(A()) == " a "
diff --git a/unittests/test_identified_cache.py b/unittests/test_identified_cache.py
index aeb5f0afcd9fc9912579bf5320bbb36b52899f07..4ed7c55c7326415308917e20e9f391b17b07ad87 100644
--- a/unittests/test_identified_cache.py
+++ b/unittests/test_identified_cache.py
@@ -27,44 +27,18 @@
test identified_cache module
"""
-from caoscrawler.identified_cache import _create_hashable_string, IdentifiedCache
import caosdb as db
-
-
-def test_create_hash():
- assert _create_hashable_string(
- db.Record("A").add_parent("B")) == "P<B>N<A>"
- assert _create_hashable_string(db.Record("A")
- .add_parent("B").add_property('a', 5)) == "P<B>N<A>a:5"
- assert (_create_hashable_string(
- db.Record("A").add_parent("B")
- .add_property('a', 4).add_property('b', 5)) == _create_hashable_string(
- db.Record("A").add_parent("B")
- .add_property('b', 5).add_property('a', 4)))
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B")
- .add_property('a', db.Record(id=12))) == "P<B>N<A>a:12")
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B")
- .add_property('a', [db.Record(id=12)])) == "P<B>N<A>a:[12]")
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B").add_property('a', [12])) == "P<B>N<A>a:[12]")
- assert (_create_hashable_string(
- db.Record("A")
- .add_parent("B")
- .add_property('a', [db.Record(id=12), 11])) == "P<B>N<A>a:[12, 11]")
- assert (_create_hashable_string(
- db.Record().add_parent("B").add_property('a', [db.Record()]))
- != _create_hashable_string(
- db.Record().add_parent("B").add_property('a', [db.Record()])))
+from caoscrawler.identifiable import Identifiable
+from caoscrawler.identified_cache import IdentifiedCache
def test_IdentifiedCache():
- ident = db.Record("A").add_parent("B")
+ ident = Identifiable(name="A", record_type="B")
record = db.Record("A").add_parent("B").add_property('b', 5)
cache = IdentifiedCache()
assert ident not in cache
cache.add(record=record, identifiable=ident)
assert ident in cache
- assert record not in cache
assert cache[ident] is record
+ assert Identifiable(name="A", record_type="C") != Identifiable(name="A", record_type="B")
+ assert Identifiable(name="A", record_type="C") not in cache
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index 6e77b0c7f26f4b2970203cfc4b8cc786fe24121b..a1724e5a989190977a7ec0d86846fc2b7433ab5d 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -22,20 +22,20 @@
from pytest import mark
+import caosdb as db
+
from caoscrawler.crawl import Crawler
-from caoscrawler.structure_elements import Dict
+from caoscrawler.identifiable import Identifiable
+from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.structure_elements import DictElement
from test_tool import rfp
-@mark.xfail(
- reason="Wait until value conversion in dicts is fixed, see "
- "https://gitlab.com/caosdb/caosdb-crawler/-/issues/10."
-)
def test_issue_10():
"""Test integer-to-float conversion in dictionaries"""
crawler_definition = {
"DictTest": {
- "type": "Dict",
+ "type": "DictElement",
"match": "(.*)",
"records": {
"TestRec": {}
@@ -63,8 +63,50 @@ def test_issue_10():
}
records = crawler.start_crawling(
- Dict("TestDict", test_dict), crawler_definition, converter_registry)
+ DictElement("TestDict", test_dict), crawler_definition, converter_registry)
assert len(records) == 1
assert records[0].parents[0].name == "TestRec"
assert records[0].get_property("float_prop") is not None
assert float(records[0].get_property("float_prop").value) == 4.0
+
+
+@mark.xfail(reason="FIX: https://gitlab.com/caosdb/caosdb-crawler/-/issues/47")
+def test_list_datatypes():
+ crawler_definition = {
+ "DictTest": {
+ "type": "DictElement",
+ "match": "(.*)",
+ "records": {
+ "Dataset": {}
+ },
+ "subtree": {
+ "int_element": {
+ "type": "IntegerElement",
+ "match_name": ".*",
+ "match_value": "(?P<int_value>.*)",
+ "records": {
+ "Dataset": {
+ "Subject": "+$int_value"
+ }
+ }
+ }
+ }
+ }
+ }
+
+ crawler = Crawler(debug=True)
+ converter_registry = crawler.load_converters(crawler_definition)
+
+ test_dict = {
+ "v1": 1233,
+ "v2": 1234
+ }
+
+ records = crawler.start_crawling(
+ DictElement("TestDict", test_dict), crawler_definition, converter_registry)
+ assert len(records) == 1
+ assert records[0].parents[0].name == "Dataset"
+ assert records[0].get_property("Subject") is not None
+ assert isinstance(records[0].get_property("Subject").value, list)
+ assert records[0].get_property("Subject").datatype is not None
+ assert records[0].get_property("Subject").datatype.startswith("LIST")
diff --git a/unittests/test_json.py b/unittests/test_json.py
index 97d9831de20a2b9f712294d1a0f6322789580f30..41fd31a43389148ad6fbc4167fd3fbd4f7f2ee9f 100644
--- a/unittests/test_json.py
+++ b/unittests/test_json.py
@@ -24,7 +24,7 @@
#
"""
-module description
+test the JSON converter
"""
import json
import os
@@ -33,7 +33,7 @@ from pytest import raises
import caosdb as db
-from caoscrawler.converters import JSONFileConverter, DictConverter
+from caoscrawler.converters import JSONFileConverter
from caoscrawler.crawl import Crawler
from caoscrawler.structure_elements import File, JSONFile
from test_tool import rfp, dircheckstr
diff --git a/unittests/test_macros.py b/unittests/test_macros.py
index 4e27e42f8d1e633cf97fa142e2c0ec8aa013af05..b5ea5d84846f5f33853910c292132d7b5026600e 100644
--- a/unittests/test_macros.py
+++ b/unittests/test_macros.py
@@ -456,3 +456,60 @@ extroot: !macro
assert cfood["extroot"]["default_name"]["a"] == "default_name"
assert cfood["extroot"]["default_name"]["v"] == "default_name"
assert cfood["extroot"]["default_name"]["macro_name"] == "default_name"
+
+
+def test_list_macro_application(register_macros, macro_store_reset):
+ dat = yaml.load("""
+defs:
+- !defmacro
+ name: test
+ params:
+ a: 2
+ definition:
+ expanded_$a:
+ param: $a
+- !defmacro
+ name: test2
+ params:
+ a: 2
+ definition:
+ expanded_${a}_test2:
+ param: $a
+
+testnode:
+ obl: !macro
+ test:
+ - a: 4
+ - a: 2
+ test2:
+ a: 4
+""", Loader=yaml.SafeLoader)
+ assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
+ assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
+ assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == "4"
+
+
+def test_variable_in_macro_definition(register_macros, macro_store_reset):
+ dat = yaml.load("""
+defs:
+- !defmacro
+ name: test
+ params:
+ a: 2
+ b: $a
+ definition:
+ expanded_$a:
+ param: $a
+ param_b: $b
+
+testnode:
+ obl: !macro
+ test:
+ - a: 4
+ - a: 2
+ b: 4
+""", Loader=yaml.SafeLoader)
+ assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
+ assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == "4"
+ assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
+ assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == "4"
diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py
index 85255d3efd34dc666d5d2e97423f33177dea6732..abe4ac85ec4fc0a78e71c177222817e1b84e9e56 100644
--- a/unittests/test_table_converter.py
+++ b/unittests/test_table_converter.py
@@ -31,10 +31,8 @@ from caoscrawler.stores import GeneralStore
from caoscrawler.converters import (ConverterValidationError,
DictConverter, XLSXTableConverter, CSVTableConverter)
from caoscrawler.structure_elements import Directory
-from caoscrawler.structure_elements import (File, DictTextElement,
- DictListElement, DictElement,
- DictBooleanElement, DictDictElement,
- DictIntegerElement, DictFloatElement)
+from caoscrawler.structure_elements import (File, TextElement, ListElement, DictElement,
+ BooleanElement, IntegerElement, FloatElement)
from os.path import join, dirname, basename
@@ -63,18 +61,17 @@ def converter_registry():
"XLSXTableConverter": {
"converter": "XLSXTableConverter",
"package": "caoscrawler.converters"},
-
- "DictDictElement": {
- "converter": "DictDictElementConverter",
+ "DictElement": {
+ "converter": "DictElementConverter",
"package": "caoscrawler.converters"},
- "DictTextElement": {
- "converter": "DictTextElementConverter",
+ "TextElement": {
+ "converter": "TextElementConverter",
"package": "caoscrawler.converters"},
- "DictIntegerElement": {
- "converter": "DictIntegerElementConverter",
+ "IntegerElement": {
+ "converter": "IntegerElementConverter",
"package": "caoscrawler.converters"},
- "DictFloatElement": {
- "converter": "DictFloatElementConverter",
+ "FloatElement": {
+ "converter": "FloatElementConverter",
"package": "caoscrawler.converters"},
}
diff --git a/unittests/test_tool.py b/unittests/test_tool.py
index 0eef86b3a9f5ef6f64d9ccb9ce0102cd87208fa4..6a828532c1de9796008a6e51c21811f83b85657a 100755
--- a/unittests/test_tool.py
+++ b/unittests/test_tool.py
@@ -1,15 +1,41 @@
-#!/bin/python
-# Tests for the tool using pytest
-# Adapted from check-sfs
-# A. Schlemmer, 06/2021
-
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Alexander Schlemmer
+# Copyright (C) 2023 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+Tests for the tool using pytest
+Adapted from check-sfs
+"""
+
+import os
from caoscrawler.crawl import Crawler, SecurityMode
+from caoscrawler.identifiable import Identifiable
from caoscrawler.structure_elements import File, DictTextElement, DictListElement
from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from simulated_server_data import full_data
from functools import partial
from copy import deepcopy
from unittest.mock import patch
+from caoscrawler.crawl import crawler_main
import caosdb.common.models as dbmodels
from unittest.mock import MagicMock, Mock
from os.path import join, dirname, basename
@@ -182,15 +208,6 @@ def test_record_structure_generation(crawler):
# ident.store_state(rfp("records.xml"))
-def test_ambigious_records(crawler, ident):
- ident.get_records().clear()
- ident.get_records().extend(crawler.crawled_data)
- r = ident.get_records()
- id_r0 = ident.get_identifiable(r[0])
- with raises(RuntimeError, match=".*unambigiously.*"):
- ident.retrieve_identified_record_for_identifiable(id_r0)
-
-
def test_crawler_update_list(crawler, ident):
# If the following assertions fail, that is a hint, that the test file records.xml has changed
# and this needs to be updated:
@@ -219,13 +236,12 @@ def test_crawler_update_list(crawler, ident):
break
id_r0 = ident.get_identifiable(r_cur)
- assert r_cur.parents[0].name == id_r0.parents[0].name
+ assert r_cur.parents[0].name == id_r0.record_type
assert r_cur.get_property(
- "first_name").value == id_r0.get_property("first_name").value
+ "first_name").value == id_r0.properties["first_name"]
assert r_cur.get_property(
- "last_name").value == id_r0.get_property("last_name").value
+ "last_name").value == id_r0.properties["last_name"]
assert len(r_cur.parents) == 1
- assert len(id_r0.parents) == 1
assert len(r_cur.properties) == 2
assert len(id_r0.properties) == 2
@@ -240,14 +256,13 @@ def test_crawler_update_list(crawler, ident):
break
id_r1 = ident.get_identifiable(r_cur)
- assert r_cur.parents[0].name == id_r1.parents[0].name
+ assert r_cur.parents[0].name == id_r1.record_type
assert r_cur.get_property(
- "identifier").value == id_r1.get_property("identifier").value
- assert r_cur.get_property("date").value == id_r1.get_property("date").value
+ "identifier").value == id_r1.properties["identifier"]
+ assert r_cur.get_property("date").value == id_r1.properties["date"]
assert r_cur.get_property(
- "project").value == id_r1.get_property("project").value
+ "project").value == id_r1.properties["project"]
assert len(r_cur.parents) == 1
- assert len(id_r1.parents) == 1
assert len(r_cur.properties) == 4
assert len(id_r1.properties) == 3
@@ -262,21 +277,6 @@ def test_crawler_update_list(crawler, ident):
"responsible").value == idr_r1.get_property("responsible").value
assert r_cur.description == idr_r1.description
- # test whether compare_entites function works in this context:
- comp = compare_entities(r_cur, id_r1)
- assert len(comp[0]["parents"]) == 0
- assert len(comp[1]["parents"]) == 0
- assert len(comp[0]["properties"]) == 1
- assert len(comp[1]["properties"]) == 0
- assert "responsible" in comp[0]["properties"]
- assert "description" in comp[0]
-
- comp = compare_entities(r_cur, idr_r1)
- assert len(comp[0]["parents"]) == 0
- assert len(comp[1]["parents"]) == 0
- assert len(comp[0]["properties"]) == 0
- assert len(comp[1]["properties"]) == 0
-
def test_synchronization(crawler, ident):
insl, updl = crawler.synchronize(commit_changes=False)
@@ -284,14 +284,6 @@ def test_synchronization(crawler, ident):
assert len(updl) == 0
-def test_identifiable_adapter():
- query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record().add_parent("Person")
- .add_property("first_name", value="A")
- .add_property("last_name", value="B"))
- assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
-
-
def test_remove_unnecessary_updates():
# test trvial case
upl = [db.Record().add_parent("A")]
@@ -359,7 +351,11 @@ def test_provenance_debug_data(crawler):
assert check_key_count("Person") == 14
-def basic_retrieve_by_name_mock_up(rec, known):
+def test_split_into_inserts_and_updates_trivial(crawler):
+ crawler.split_into_inserts_and_updates([])
+
+
+def basic_retrieve_by_name_mock_up(rec, referencing_entities=None, known=None):
""" returns a stored Record if rec.name is an existing key, None otherwise """
if rec.name in known:
return known[rec.name]
@@ -377,27 +373,26 @@ def crawler_mocked_identifiable_retrieve(crawler):
# There is only a single known Record with name A
crawler.identifiableAdapter.retrieve_identified_record_for_record = Mock(side_effect=partial(
basic_retrieve_by_name_mock_up, known={"A": db.Record(id=1111, name="A")}))
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable = Mock(
+ side_effect=partial(
+ basic_retrieve_by_name_mock_up, known={"A": db.Record(id=1111, name="A")}))
return crawler
-def test_split_into_inserts_and_updates_trivial(crawler):
- # Try trivial argument
- crawler.split_into_inserts_and_updates([])
-
-
def test_split_into_inserts_and_updates_single(crawler_mocked_identifiable_retrieve):
crawler = crawler_mocked_identifiable_retrieve
+ identlist = [Identifiable(name="A", record_type="C"), Identifiable(name="B", record_type="C")]
entlist = [db.Record(name="A").add_parent(
"C"), db.Record(name="B").add_parent("C")]
- assert crawler.get_from_any_cache(entlist[0]) is None
- assert crawler.get_from_any_cache(entlist[1]) is None
- assert not crawler.has_reference_value_without_id(entlist[0])
- assert not crawler.has_reference_value_without_id(entlist[1])
+ assert crawler.get_from_any_cache(identlist[0]) is None
+ assert crawler.get_from_any_cache(identlist[1]) is None
+ assert not crawler._has_reference_value_without_id(identlist[0])
+ assert not crawler._has_reference_value_without_id(identlist[1])
assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
- entlist[0]).id == 1111
+ identlist[0]).id == 1111
assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
- entlist[1]) is None
+ identlist[1]) is None
insert, update = crawler.split_into_inserts_and_updates(deepcopy(entlist))
assert len(insert) == 1
@@ -406,7 +401,7 @@ def test_split_into_inserts_and_updates_single(crawler_mocked_identifiable_retri
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_duplicate(crawler_mocked_identifiable_retrieve):
@@ -424,7 +419,7 @@ def test_split_into_inserts_and_updates_with_duplicate(crawler_mocked_identifiab
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_ref(crawler_mocked_identifiable_retrieve):
@@ -440,8 +435,8 @@ def test_split_into_inserts_and_updates_with_ref(crawler_mocked_identifiable_ret
assert len(update) == 1
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_circ(crawler):
@@ -476,7 +471,7 @@ def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
# TODO write test where the unresoled entity is not part of the identifiable
@@ -495,7 +490,7 @@ def test_split_into_inserts_and_updates_with_copy_attr(crawler_mocked_identifiab
assert update[0].get_property("foo").value == 1
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_has_missing_object_in_references(crawler):
@@ -509,56 +504,56 @@ def test_has_missing_object_in_references(crawler):
}))
# one reference with id -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C")
- .add_parent("RTC").add_property('d', 123))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': 123}), [])
# one ref with Entity with id -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C")
- .add_parent("RTC")
- .add_property('d', db.Record(id=123)
- .add_parent("C")))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': db.Record(id=123)
+ .add_parent("C")}), [])
# one ref with id one with Entity with id (mixed) -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C").add_parent("RTD")
- .add_property('d', 123)
- .add_property('b', db.Record(id=123)
- .add_parent("RTC")))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTD",
+ properties={'d': 123, 'b': db.Record(id=123).add_parent("RTC")}), [])
# entity to be referenced in the following
a = db.Record(name="C").add_parent("C").add_property("d", 12311)
# one ref with id one with Entity without id (but not identifying) -> fail
- assert not crawler.has_missing_object_in_references(db.Record(name="C").add_parent("RTC")
- .add_property('d', 123)
- .add_property('e', a))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': 123, 'e': a}), [])
# one ref with id one with Entity without id (mixed) -> fail
- assert not crawler.has_missing_object_in_references(db.Record(name="D").add_parent("RTD")
- .add_property('d', 123)
- .add_property('e', a))
- crawler.add_to_remote_missing_cache(a)
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="D", record_type="RTD", properties={'d': 123, 'e': a}), [])
+
+ crawler.add_to_remote_missing_cache(a, Identifiable(name="C", record_type="RTC",
+ properties={'d': 12311}))
# one ref with id one with Entity without id but in cache -> check
- assert crawler.has_missing_object_in_references(db.Record(name="D").add_parent("RTD")
- .add_property('d', 123)
- .add_property('e', a))
+ assert crawler._has_missing_object_in_references(
+ Identifiable(name="D", record_type="RTD", properties={'d': 123, 'e': a}), [])
+
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
+@pytest.mark.xfail()
def test_references_entities_without_ids(crawler, ident):
- assert not crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('last_name', 123)
- .add_property('first_name', 123))
+ assert not crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('last_name', 123)
+ .add_property('first_name', 123))
# id and rec with id
- assert not crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('last_name', db.Record(id=123)))
+ assert not crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('last_name',
+ db.Record(id=123)))
# id and rec with id and one unneeded prop
- assert crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('stuff', db.Record())
- .add_property('last_name', db.Record(id=123)))
+ assert crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('stuff', db.Record())
+ .add_property('last_name', db.Record(id=123)))
# one identifying prop is missing
- assert crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('last_name', db.Record()))
+ assert crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('last_name', db.Record()))
def test_replace_entities_with_ids(crawler):
@@ -604,24 +599,37 @@ def reset_mocks(mocks):
def change_identifiable_prop(ident):
- # the checks in here are only to make sure we change the record as we intend to
- meas = ident._records[-2]
- assert meas.parents[0].name == "Measurement"
- resps = meas.properties[0]
- assert resps.name == "date"
- # change one element; This changes the date which is part of the identifiable
- resps.value = "2022-01-04"
+ """
+ This function is supposed to change a non identifiing property.
+ """
+ for ent in ident._records:
+ if len(ent.parents) == 0 or ent.parents[0].name != "Measurement":
+ continue
+ for prop in ent.properties:
+ if prop.name != "date":
+ continue
+ # change one element; This removes a responsible which is not part of the identifiable
+ prop.value = "2022-01-04"
+ return
+ # If it does not work, this test is not implemented properly
+ raise RuntimeError("Did not find the property that should be changed.")
def change_non_identifiable_prop(ident):
- # the checks in here are only to make sure we change the record as we intend to
- meas = ident._records[-1]
- assert meas.parents[0].name == "Measurement"
- resps = meas.properties[-1]
- assert resps.name == "responsible"
- assert len(resps.value) == 2
- # change one element; This removes a responsible which is not part of the identifiable
- del resps.value[-1]
+ """
+ This function is supposed to change a non identifiing property.
+ """
+ for ent in ident._records:
+ if len(ent.parents) == 0 or ent.parents[0].name != "Measurement":
+ continue
+
+ for prop in ent.properties:
+ if prop.name != "responsible" or len(prop.value) < 2:
+ continue
+ # change one element; This removes a responsible which is not part of the identifiable
+ del prop.value[-1]
+ return
+ raise RuntimeError("Did not find the property that should be changed.")
@patch("caoscrawler.crawl.Crawler._get_entity_by_id",
@@ -715,7 +723,147 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
ident._records = deepcopy(records_backup)
+def test_create_reference_mapping():
+ a = db.Record().add_parent("A")
+ b = db.Record().add_parent("B").add_property('a', a)
+ ref = Crawler.create_reference_mapping([a, b])
+ assert id(a) in ref
+ assert id(b) not in ref
+ assert "B" in ref[id(a)]
+ assert ref[id(a)]["B"] == [b]
+
+
def test_create_flat_list():
a = db.Record()
+ b = db.Record()
a.add_property(name="a", value=a)
- Crawler.create_flat_list([a], [])
+ a.add_property(name="b", value=b)
+ flat = Crawler.create_flat_list([a])
+ assert len(flat) == 2
+ assert a in flat
+ assert b in flat
+ c = db.Record()
+ c.add_property(name="a", value=a)
+ # This would caus recursion if it is not dealt with properly.
+ a.add_property(name="c", value=c)
+ flat = Crawler.create_flat_list([c])
+ assert len(flat) == 3
+ assert a in flat
+ assert b in flat
+ assert c in flat
+
+
+@pytest.fixture
+def crawler_mocked_for_backref_test(crawler):
+ # mock retrieval of registered identifiabls: return Record with just a parent
+ def get_reg_ident(x):
+ if x.parents[0].name == "C":
+ return db.Record().add_parent(x.parents[0].name).add_property(
+ "is_referenced_by", value=["BR"])
+ elif x.parents[0].name == "D":
+ return db.Record().add_parent(x.parents[0].name).add_property(
+ "is_referenced_by", value=["BR", "BR2"])
+ else:
+ return db.Record().add_parent(x.parents[0].name)
+ crawler.identifiableAdapter.get_registered_identifiable = Mock(side_effect=get_reg_ident)
+
+ # Simulate remote server content by using the names to identify records
+ # There is only a single known Record with name A
+ crawler.identifiableAdapter.retrieve_identified_record_for_record = Mock(side_effect=partial(
+ basic_retrieve_by_name_mock_up, known={"A":
+ db.Record(id=1111, name="A").add_parent("BR")}))
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable = Mock(
+ side_effect=partial(
+ basic_retrieve_by_name_mock_up, known={"A":
+ db.Record(id=1111, name="A").add_parent("BR")}))
+ return crawler
+
+
+def test_validation_error_print(capsys):
+ # there should be no server interaction since we only test the behavior if a validation error
+ # occurs during the data collection stage
+ DATADIR = os.path.join(os.path.dirname(__file__), "test_data", "failing_validation")
+ for fi in ["cfood.yml", "cfood2.yml"]:
+ ret = crawler_main(DATADIR,
+ os.path.join(DATADIR, fi),
+ os.path.join(DATADIR, "identifiables.yml"),
+ True,
+ None,
+ False,
+ "/use_case_simple_presentation")
+ captured = capsys.readouterr()
+ assert "Couldn't validate" in captured.out
+
+
+def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test):
+ crawler = crawler_mocked_for_backref_test
+ identlist = [Identifiable(name="A", record_type="BR"),
+ Identifiable(name="B", record_type="C", backrefs=[db.Entity()])]
+ referenced = db.Record(name="B").add_parent("C")
+ entlist = [referenced, db.Record(name="A").add_parent("BR").add_property("ref", referenced), ]
+
+ # Test without referencing object
+ # currently a NotImplementedError is raised if necessary properties are missing.
+ with raises(NotImplementedError):
+ crawler.split_into_inserts_and_updates([db.Record(name="B").add_parent("C")])
+
+ # identifiables were not yet checked
+ assert crawler.get_from_any_cache(identlist[0]) is None
+ assert crawler.get_from_any_cache(identlist[1]) is None
+ # one with reference, one without
+ assert not crawler._has_reference_value_without_id(identlist[0])
+ assert crawler._has_reference_value_without_id(identlist[1])
+ # one can be found remotely, one not
+ assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
+ identlist[0]).id == 1111
+ assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
+ identlist[1]) is None
+
+ # check the split...
+ insert, update = crawler.split_into_inserts_and_updates(deepcopy(entlist))
+ # A was found remotely and is therefore in the update list
+ assert len(update) == 1
+ assert update[0].name == "A"
+ # B does not exist on the (simulated) remote server
+ assert len(insert) == 1
+ assert insert[0].name == "B"
+
+
+def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_test):
+ # test whether multiple references of the same record type are correctly used
+ crawler = crawler_mocked_for_backref_test
+ referenced = db.Record(name="B").add_parent("C")
+ entlist = [referenced,
+ db.Record(name="A").add_parent("BR").add_property("ref", referenced),
+ db.Record(name="C").add_parent("BR").add_property("ref", referenced),
+ ]
+
+ # test whether both entities are listed in the backref attribute of the identifiable
+ referencing_entities = crawler.create_reference_mapping(entlist)
+ identifiable = crawler.identifiableAdapter.get_identifiable(referenced, referencing_entities)
+ assert len(identifiable.backrefs) == 2
+
+ # check the split...
+ insert, update = crawler.split_into_inserts_and_updates(deepcopy(entlist))
+ assert len(update) == 1
+ assert len(insert) == 2
+
+
+def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_test):
+ # test whether multiple references of the different record types are correctly used
+ crawler = crawler_mocked_for_backref_test
+ referenced = db.Record(name="B").add_parent("D")
+ entlist = [referenced,
+ db.Record(name="A").add_parent("BR").add_property("ref", referenced),
+ db.Record(name="A").add_parent("BR2").add_property("ref", referenced),
+ ]
+
+ # test whether both entities are listed in the backref attribute of the identifiable
+ referencing_entities = crawler.create_reference_mapping(entlist)
+ identifiable = crawler.identifiableAdapter.get_identifiable(referenced, referencing_entities)
+ assert len(identifiable.backrefs) == 2
+
+ # check the split...
+ insert, update = crawler.split_into_inserts_and_updates(deepcopy(entlist))
+ assert len(update) == 2
+ assert len(insert) == 1
diff --git a/unittests/test_validation.py b/unittests/test_validation.py
deleted file mode 100644
index 686c66f72f55b66344322e0c6f3b9d1a2b76b3f9..0000000000000000000000000000000000000000
--- a/unittests/test_validation.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-# encoding: utf-8
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2022 Alexander Schlemmer
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-
-"""
-Test the validation of cfood definition files.
-"""
-
-from caoscrawler.crawl import Crawler
-
-from tempfile import NamedTemporaryFile
-
-import yaml
-import pytest