initial commit

# Byte-compiled / optimized / DLL files

__pycache__/

*.py[cod]

*$py.class

# C extensions

*.so

# Distribution / packaging

.Python

build/

develop-eggs/

dist/

downloads/

eggs/

.eggs/

lib/

lib64/

parts/

sdist/

var/

wheels/

share/python-wheels/

*.egg-info/

.installed.cfg

*.egg

MANIFEST

# PyInstaller

#  Usually these files are written by a python script from a template

#  before PyInstaller builds the exe, so as to inject date/other infos into it.

*.manifest

*.spec

# Installer logs

pip-log.txt

pip-delete-this-directory.txt

# Unit test / coverage reports

htmlcov/

.tox/

.nox/

.coverage

.coverage.*

.cache

nosetests.xml

coverage.xml

*.cover

*.py,cover

.hypothesis/

.pytest_cache/

cover/

# Translations

*.mo

*.pot

# Django stuff:

*.log

local_settings.py

db.sqlite3

db.sqlite3-journal

# Flask stuff:

instance/

.webassets-cache

# Scrapy stuff:

.scrapy

# Sphinx documentation

docs/_build/

# PyBuilder

.pybuilder/

target/

# Jupyter Notebook

.ipynb_checkpoints

# IPython

profile_default/

ipython_config.py

# pyenv

#   For a library or package, you might want to ignore these files since the code is

#   intended to run in multiple environments; otherwise, check them in:

# .python-version

# pipenv

#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.

#   However, in case of collaboration, if having platform-specific dependencies or dependencies

#   having no cross-platform support, pipenv may install dependencies that don't work, or not

#   install all needed dependencies.

#Pipfile.lock

# poetry

#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.

#   This is especially recommended for binary packages to ensure reproducibility, and is more

#   commonly ignored for libraries.

#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control

#poetry.lock

# PEP 582; used by e.g. github.com/David-OConnor/pyflow

__pypackages__/

# Celery stuff

celerybeat-schedule

celerybeat.pid

# SageMath parsed files

*.sage.py

# Environments

.env

.venv

env/

venv/

ENV/

env.bak/

venv.bak/

# Spyder project settings

.spyderproject

.spyproject

# Rope project settings

.ropeproject

# mkdocs documentation

/site

# mypy

.mypy_cache/

.dmypy.json

dmypy.json

# Pyre type checker

.pyre/

# pytype static type analyzer

.pytype/

# Cython debug symbols

cython_debug/

# PyCharm

#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can

#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore

#  and can be added to the global gitignore or merged into this file.  For a more nuclear

#  option (not recommended) you can uncomment the following to ignore the entire idea folder.

#.idea/

venv/

*.json

*.db

*.zip

.ruff_cache/

data/*.parquet

experiments/

 No newline at end of file

# dois2SQLite

## `dois2sqlite`

Tool for loading Crossref metadata into a SQLite database

**Installation**

## Getting started

- `python -m venv venv`

- `. ./venv/bin/activate`

- `pip install -e .` (`-e` assuming you might want to edit package)

To make it easy for you to get started with GitLab, here's a list of recommended next steps.

**Usage**:

Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!

```console

$ dois2sqlite [OPTIONS] COMMAND [ARGS]...

```

## Add your files

**Options**:

- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files

- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:

- `--install-completion`: Install completion for the current shell.

- `--show-completion`: Show completion for the current shell, to copy it or customize the installation.

- `--help`: Show this message and exit.

```

cd existing_repo

git remote add origin https://gitlab.com/crossref/labs/dois2sqlite.git

git branch -M main

git push -uf origin main

**Commands**:

- `create`: Create a new SQLite database at the...

- `index`: Create indexes on an existing SQLite...

- `load`: Load data from a src into SQLite.

- `tarinfo`: Analyze the TAR file and print out...

## `dois2sqlite create`

Create a new SQLite database at the specified path.

**Usage**:

```console

$ dois2sqlite create [OPTIONS] SQLITE_PATH

```

## Integrate with your tools

**Arguments**:

- [ ] [Set up project integrations](https://gitlab.com/crossref/labs/dois2sqlite/-/settings/integrations)

- `SQLITE_PATH`: [required]

## Collaborate with your team

**Options**:

- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)

- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)

- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)

- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)

- [ ] [Set auto-merge](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)

- `--verbose / --no-verbose`: Verbose mode [default: no-verbose]

- `--help`: Show this message and exit.

## Test and Deploy

## `dois2sqlite index`

Use the built-in continuous integration in GitLab.

Create indexes on an existing SQLite database.

- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)

- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)

- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)

- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)

- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)

**Usage**:

***

```console

$ dois2sqlite index [OPTIONS] SQLITE_PATH

```

**Arguments**:

- `SQLITE_PATH`: [required]

# Editing this README

**Options**:

When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template.

- `--verbose / --no-verbose`: Verbose mode [default: no-verbose]

- `--help`: Show this message and exit.

## Suggestions for a good README

## `dois2sqlite load`

Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.

Load data from a src into SQLite.

## Name

Choose a self-explaining name for your project.

**Usage**:

## Description

Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.

```console

$ dois2sqlite load [OPTIONS] SRC_PATH SQLITE_PATH

```

**Arguments**:

## Badges

On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.

- `SRC_PATH`: [required]

- `SQLITE_PATH`: [required]

## Visuals

Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.

**Options**:

## Installation

Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.

- `--n-jobs INTEGER`: Number of jobs [default: 1]

- `--commit-size INTEGER`: Number of records to commit at a time [default: 500000]

- `--verbose / --no-verbose`: Verbose mode [default: no-verbose]

- `--dry-run / --no-dry-run`: Dry run. Does everything except the actual insert into the database. [default: no-dry-run]

- `--max-files INTEGER`: Maximum number of files to process

- `--convert-to-commonmeta / --no-convert-to-commonmeta`: Convert to commonmeta [default: no-convert-to-commonmeta]

- `--clobber-sqlite / --no-clobber-sqlite`: Clobber the SQLite database, if it already exists [default: no-clobber-sqlite]

- `--help`: Show this message and exit.

## Usage

Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.

## `dois2sqlite tarinfo`

## Support

Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.

Analyze the TAR file and print out information about it, including the following:

## Roadmap

If you have ideas for releases in the future, it is a good idea to list them in the README.

- The number of JSON files in the TAR file

- The estimated number of work items in total

## Contributing

State if you are open to contributions and what your requirements are for accepting them.

**Usage**:

For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.

```console

$ dois2sqlite tarinfo [OPTIONS] TAR_PATH

```

You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.

**Arguments**:

## Authors and acknowledgment

Show your appreciation to those who have contributed to the project.

- `TAR_PATH`: [required]

## License

For open source projects, say how it is licensed.

**Options**:

## Project status

If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.

- `--verbose / --no-verbose`: Verbose mode [default: no-verbose]

- `--help`: Show this message and exit.

import json

import logging

import os

import gzip

import sqlite3

import tarfile

from itertools import islice

from pathlib import Path

from time import time

from typing import Tuple, List, Optional, Generator

import typer

from joblib import Parallel, delayed

from commonmeta.readers.crossref_reader import read_crossref

logging.basicConfig(level=logging.INFO)

logger = logging.getLogger(__name__)

app = typer.Typer(help="Tool for loading Crossref metadata into a SQLite database")

VERBOSE_HELP = "Verbose mode"

PARALLEL_PREFER = "processes"

#DEFAULT_NUMBERS_OF_JOBS = 1

DEFAULT_COMMIT_SIZE = 500000

TEMP_DIR = "/tmp/cr-records"

# return the number of CPU cores

def get_cpu_count() -> int:

    try:

        return os.cpu_count()

    except AttributeError:

        return 1

def batched(iterable, n) -> Generator[Tuple, None, None]:

    """Batch an iterable into chunks of size n

    NB this is built into Python 3.12 as `itertools.batched`

    """

    # batched('ABCDEFG', 3) --> ABC DEF G

    if n < 1:

        raise ValueError("n must be at least one")

    it = iter(iterable)

    while batch := tuple(islice(it, n)):

        yield batch

def all_items(fn) -> List[dict]:

    with open(fn, "rb") as f:

        return json.load(f)["items"]

def json_files_in_tar(src_path) -> List[str]:

    with tarfile.open(src_path, "r") as tar:

        return [

            member.name

            for member in tar.getmembers()

            if member.isfile() and member.name.endswith(".json")

        ]

### SQLite stuff

def create_table(cursor) -> None:

    """Create the table with the specified schema."""

    cursor.execute(

        """

        CREATE TABLE IF NOT EXISTS works (

            id INTEGER PRIMARY KEY AUTOINCREMENT,

            doi VARCHAR(255),

            resource_primary_url VARCHAR(255),

            type VARCHAR(255),

            member INTEGER,

            prefix VARCHAR(8),

            created_date_time DATE,

            deposited_date_time DATE,

            commonmeta_format BOOLEAN,

            metadata BLOB

        )

    """

    )

def insert_records(cursor, records) -> None:

    """Insert a list of records into the database."""

    logger.info(f"Inserting {len(records):,} records")

    t1 = time()

    cursor.executemany(

        """

        INSERT INTO works (doi, resource_primary_url, type, member, prefix, created_date_time, deposited_date_time, commonmeta_format, metadata) 

        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)

    """,

        records,

    )

    t2 = time()

    logger.info(f"Insert took {t2 - t1:.2f} seconds")

    logger.info("Committing")

    cursor.connection.commit()

    t3 = time()

    logger.info(f"Commit took {t3 - t2:.2f} seconds")

def convert_to_common(item) -> Tuple[Optional[bool], List[str]]:

    try:

        commonmeta = read_crossref(item)

        return True, commonmeta

    except Exception as e:

        logging.error(

            f"Error converting to commonmeta: DOI:{item.get('DOI', '').lower()}: {e}"

        )

        return False , item

def generate_record(item, convert_to_commonmeta) -> Tuple[str, str, str, int, str, str, str, bool, str]:

    converted = False

    if convert_to_commonmeta:

        converted, item = convert_to_common(item)

    return (

        item.get("DOI", "").lower(),

        item.get("resource", {}).get("primary", {}).get("URL", ""),

        item.get("type", ""),

        item.get("member", 0),

        item.get("prefix", ""),

        item.get("created", {}).get("date-time", ""),

        item.get("deposited", {}).get("date-time", ""),

        converted,

        f"{json.dumps(item)}",

    )

def create_records_from_tar(file_name, tar_file_path, convert_to_commonmeta) -> List[Tuple[str, str, str, int, str, str, str, bool, str]]:

    logger.info(f"Extracting records from: {file_name}")

    t1 = time()

    with tarfile.open(tar_file_path) as tar:

        member = tar.getmember(file_name)

        fn = Path(TEMP_DIR, member.name)

        tar.extract(member, path=TEMP_DIR)

        records = [

            generate_record(item, convert_to_commonmeta) for item in all_items(fn)

        ]

        fn.unlink()

    t2 = time()

    logger.info(

        f"Extracted {len(records):,} records from {file_name} in {t2 - t1:.2f} seconds"

    )

    return records

def create_records_from_compressed_json(compressed_json_file, _, convert_to_commonmeta) -> List[Tuple[str, str, str, int, str, str, str, bool, str]]:

    logger.info(f"Extracting records from: {compressed_json_file}")

    t1 = time()

    with gzip.open(compressed_json_file, "rb") as f:

        records = [

            generate_record(item, convert_to_commonmeta)

            for item in json.load(f)["items"]

        ]

    t2 = time()

    logger.info(

        f"Extracted {len(records):,} records from {compressed_json_file} in {t2 - t1:.2f} seconds"

    )

    return records

def collect_json_files(src_path, max_files) -> Tuple[List[str], callable]:

    if src_path.is_dir():

        jason_files = (

            list(src_path.glob("*.json.gz"))[:max_files]

            if max_files

            else list(src_path.glob("*.json.gz"))

        )

        create_records_func = create_records_from_compressed_json

    elif src_path.is_file():

        jason_files = (

            json_files_in_tar(src_path)[:max_files]

            if max_files

            else json_files_in_tar(src_path)

        )

        create_records_func = create_records_from_tar

    return jason_files, create_records_func

def process_files_and_insert_records(

    jason_files,

    create_records_func,

    src_path,

    sqlite_path,

    n_jobs,

    commit_size,

    dry_run,

    convert_to_commonmeta,

) -> int:

    total_records = 0

    with sqlite3.connect(sqlite_path) as conn:

        cursor = conn.cursor()

        commit_batch = []

        for file_batch in batched(jason_files, n_jobs):

            results = Parallel(n_jobs=n_jobs, prefer=PARALLEL_PREFER)(

                delayed(create_records_func)(json_file, src_path, convert_to_commonmeta)

                for json_file in file_batch

            )

            commit_batch.extend(item for sublist in results for item in sublist)

            if len(commit_batch) >= commit_size:

                total_records += len(commit_batch)

                if not dry_run:

                    insert_records(cursor, commit_batch)

                logger.info(f"Current total: {total_records:,}")

                commit_batch = []

        if commit_batch and not dry_run:

            insert_records(cursor, commit_batch)

            total_records += len(commit_batch)

    return total_records

@app.command()

def create(sqlite_path: Path = typer.Argument(..., exists=False), verbose: bool = typer.Option(False, help=VERBOSE_HELP)) -> None:

    """

    Create a new SQLite database at the specified path.

    """

    if verbose:

        logger.setLevel(logging.INFO)

        logger.info("Verbose mode")

    with sqlite3.connect(sqlite_path) as conn:

        cursor = conn.cursor()

        create_table(cursor)

    logger.info(f"Database created at {sqlite_path}")

@app.command()

def load(

    src_path: Path = typer.Argument(..., exists=True),

    sqlite_path: Path = typer.Argument(..., exists=True),

    n_jobs: int = typer.Option(1, help="Number of jobs"),

    commit_size: int = typer.Option(

        DEFAULT_COMMIT_SIZE, help="Number of records to commit at a time"

    ),

    verbose: bool = typer.Option(False, help=VERBOSE_HELP),

    dry_run: bool = typer.Option(

        False,

        help="Dry run. Does everything except the actual insert into the database.",

    ),

    max_files: int = typer.Option(None, help="Maximum number of files to process"),

    convert_to_commonmeta: bool = typer.Option(False, help="Convert to commonmeta"),

    clobber_sqlite: bool = typer.Option(False, help="Clobber the SQLite database, if it already exists"),

) -> None:

    """

    Load data from a src into SQLite.

    """

    if verbose:

        logger.setLevel(logging.INFO)

        logger.info("Verbose mode")

    jason_files, create_records_func = collect_json_files(src_path, max_files)

    total_records = process_files_and_insert_records(

        jason_files,

        create_records_func,

        src_path,

        sqlite_path,

        n_jobs,

        commit_size,

        dry_run,

        convert_to_commonmeta,

    )

    logger.info(

        f"Done: {total_records:,} records loaded from {src_path} into {sqlite_path}"

    )

@app.command()

def index(sqlite_path: Path = typer.Argument(..., exists=True), verbose: bool = typer.Option(False, help=VERBOSE_HELP)) -> None:

    """

    Create indexes on an existing SQLite database.

    """

    if verbose:

        logger.setLevel(logging.INFO)

        logger.info("Verbose mode")

    with sqlite3.connect(sqlite_path) as conn:

        cursor = conn.cursor()

        indexes = [

            "doi",

            # "resource_primary_url",

            "type",

            "member",

            "prefix",

            "created_date_time",

            "deposited_date_time",

            "commonmeta_format",

        ]

    for index in indexes:

        logger.info(f"Creating index on {index}")

        t1 = time()

        cursor.execute(f"CREATE INDEX IF NOT EXISTS idx_{index} ON works ({index})")

        logger.info(f"Index on {index} took {time() - t1:.2f} seconds")

    logger.info("Committing indexes")

    t1 = time()

    cursor.connection.commit()

    logger.info(f"Index commit took {time() - t1:.2f} seconds")

    logger.info(f"Indexes created on the database at {sqlite_path}")

@app.command()

def tarinfo(tar_path: Path = typer.Argument(..., exists=True), verbose: bool = typer.Option(False,help=VERBOSE_HELP)) -> None:

    """

    Analyze the TAR file and print out information about it, including the following:

    - The number of JSON files in the TAR file

    - The estimated number of work items in total

    """

    if verbose:

        logger.setLevel(logging.INFO)

        logger.info("Verbose mode")

    with tarfile.open(tar_path, "r") as tar_file:

        number_of_files = len(json_files_in_tar(tar_file))

        estimated_number_of_items = number_of_files * 5000

        print(f"Number of files: {number_of_files}")

        print(f"Estimated number of items: {estimated_number_of_items:,}")

        print(f"Recommended cores: {get_cpu_count()}")

if __name__ == "__main__":

    app()

[project]

# name: String, REQUIRED

name = "dois2sqlite"

version = "0.1"

description = "A tools for loading Crossref snapshots or public data files into a SQLite database."

readme = "README.md"

requires-python = ">=3.11"

license = {text = "MIT"}

authors = [

    { name = "Geoffrey Bilder", email = "gbilder@crossref.org" },

    { name = "Crossref Labs", email = "labs@crossref.org" },

]

# urls: Table

# Should describe where to find useful info for your project

urls = { source = "https://gitlab.com/crossref/labs/dois2sqlite", documentation = "https://crossref.org", homepage = "https://gitlab.com/crossref/labs/dois2sqlite", issues = "https://gitlab.com/crossref/labs/dois2sqlite/issues", ci = "https://gitlab.com/crossref/labs/dois2sqlite/pipelines" }

# dependencies: Array of Strings

# A list of requirements for our package

dependencies = [

    "typer",

    "commonmeta-py",

    "joblib",

]

[project.optional-dependencies]

dev = [

    "pytest",

    "pytest-asyncio",

    "mypy",

    "black",

    "ruff",

]

doc = ["sphinx"]

[tool.setuptools.packages.find]

include = ["dois2sqlite"]

# entry points

[project.scripts]

dois2sqlite = "dois2sqlite.cli:app"