Skip to content

Feature/sgr cloud seed changes #606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mildbyte merged 2 commits into from
Jan 6, 2022
Merged

Feature/sgr cloud seed changes #606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ab2193d
Add UIDs to seeds
mildbyte Jan 6, 2022
f1a436f
Pass the seed to the `externalPlugins` API
mildbyte Jan 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions splitgraph/cloud/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -948,9 +948,9 @@ def _make_plugin(plugin_dict: Dict[str, Any]) -> Plugin:
supports_mount=plugin_dict["supportsMount"],
)

def get_all_plugins(self) -> List[Plugin]:
def get_all_plugins(self, seed: Optional[str] = None) -> List[Plugin]:
response = self._gql(
{"query": GET_PLUGINS, "operationName": "ExternalPlugins"},
{"query": GET_PLUGINS, "operationName": "ExternalPlugins", "variables": {"seed": seed}},
handle_errors=True,
anonymous_ok=True,
)
Expand Down
21 changes: 16 additions & 5 deletions splitgraph/cloud/project/generation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import base64
import itertools
import os
import random
import string
from io import StringIO
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Tuple

import ruamel.yaml
from pydantic import BaseModel
from pydantic import BaseModel, Field
from ruamel.yaml import CommentedMap as CM
from ruamel.yaml import CommentedSeq as CS

Expand Down Expand Up @@ -143,12 +145,20 @@ def stub_plugin(plugin: Plugin, namespace: str, repository: str, is_live: bool =
return ruamel_dict


def _get_seed_uid() -> str:
return "".join(
random.choice(string.ascii_lowercase + string.ascii_uppercase + string.digits)
for _ in range(10)
)


class ProjectSeed(BaseModel):
"""
Contains all information required to generate a Splitgraph project + optionally
a dbt model for GitHub Actions
"""

seed_uid: str = Field(default_factory=_get_seed_uid)
namespace: str
plugins: List[str]
include_dbt: bool = False
Expand All @@ -162,11 +172,12 @@ def decode(cls, encoded: str) -> "ProjectSeed":


def generate_project(
api_client: GQLAPIClient, seed: ProjectSeed, basedir: Path, github_repo: Optional[str] = None
api_client: GQLAPIClient, seed: str, basedir: Path, github_repo: Optional[str] = None
) -> None:
all_plugins = {p.plugin_name: p for p in api_client.get_all_plugins()}
all_plugins = {p.plugin_name: p for p in api_client.get_all_plugins(seed)}

credentials, repositories, repository_info = generate_splitgraph_yml(all_plugins, seed)
decoded_seed = ProjectSeed.decode(seed)
credentials, repositories, repository_info = generate_splitgraph_yml(all_plugins, decoded_seed)

yml = ruamel.yaml.YAML()
with open(os.path.join(basedir, "splitgraph.credentials.yml"), "w") as f:
Expand All @@ -176,7 +187,7 @@ def generate_project(
yml.dump(repositories, f)

# Generate the dbt project
if seed.include_dbt:
if decoded_seed.include_dbt:
dbt_repo, _, is_dbt = repository_info[-1]
assert is_dbt
dbt_sources = [r for r, _, is_dbt in repository_info if not is_dbt]
Expand Down
4 changes: 2 additions & 2 deletions splitgraph/cloud/queries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@
}
"""

GET_PLUGINS = """query ExternalPlugins {
externalPlugins {
GET_PLUGINS = """query ExternalPlugins($seed: String) {
externalPlugins(seed: $seed) {
pluginName
name
description
Expand Down
4 changes: 2 additions & 2 deletions splitgraph/commandline/cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1179,11 +1179,11 @@ def validate_c(repositories_file):
def seed_c(remote, seed, github_repository, directory):
"""Generate a starter Splitgraph Cloud project from a seed."""
from splitgraph.cloud import GQLAPIClient
from splitgraph.cloud.project.generation import ProjectSeed, generate_project
from splitgraph.cloud.project.generation import generate_project

client = GQLAPIClient(remote)

generate_project(client, ProjectSeed.decode(seed), directory, github_repo=github_repository)
generate_project(client, seed, directory, github_repo=github_repository)
click.echo(f"Splitgraph project generated in {os.path.abspath(directory)}.")


Expand Down
6 changes: 4 additions & 2 deletions test/splitgraph/cloud/project/test_generation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ def test_generate_project_no_dbt(snapshot):
), TemporaryDirectory() as tmpdir:
generate_project(
GQLAPIClient("anyremote"),
seed=ProjectSeed(namespace="myns", plugins=["postgres_fdw", "airbyte-postgres"]),
seed=ProjectSeed(
namespace="myns", plugins=["postgres_fdw", "airbyte-postgres"]
).encode(),
basedir=tmpdir,
github_repo="my/repo",
)
Expand Down Expand Up @@ -63,7 +65,7 @@ def test_generate_project_with_dbt(snapshot):
GQLAPIClient("anyremote"),
seed=ProjectSeed(
namespace="myns", plugins=["postgres_fdw", "airbyte-postgres"], include_dbt=True
),
).encode(),
basedir=tmpdir,
github_repo="my/repo",
)
Expand Down