pax_global_header00006660000000000000000000000064145145452650014525gustar00rootroot0000000000000052 comment=1c41524d53f4007889bc5101ad39b6f2a2d84c36 python-distribution-parser-0.2.3/000077500000000000000000000000001451454526500170575ustar00rootroot00000000000000python-distribution-parser-0.2.3/README.md000066400000000000000000000324701451454526500203440ustar00rootroot00000000000000# Python Distribution Parser Have you ever wanted to parse Python distributions like `.whl` files from Go before? Wait, you haven't? Huh. ## Purpose The main goal is to provide the ability to parse the metadata of a Python distribution. This almost entirely matches what `twine` does, feel free to compare the code here: https://github.com/pypa/twine/blob/afc37f8b26ed06ccd104f6724f293f657b9b7f15/twine/commands/upload.py#L93 This implementation supports parsing only `.tar.gz` and `.whl` files (also know as `sdist` and `bdist_whl`). Currently, `.exe` (`bdist_wininst`), and `.egg` (`bdist_egg`) are not supported. ## Installation ```sh $ go get github.com/rstudio/python-distribution-parser ``` ## Usage The main entrypoint is the `Parse(path string) ([]*packages.PackageFile, error)` function. This takes in a file or directory path and will return an array of `*PackageFile` objects based on the distribution(s) passed in. Here it is in action: ```go package main import ( "encoding/json" "fmt" pdp "github.com/rstudio/python-distribution-parser" ) func main() { filePath := "/path/to/appdirs/dist/appdirs-1.4.4-py2.py3-none-any.whl" packages, err := pdp.Parse(filePath) if err != nil { log.Fatalf("Error parsing directory %s: %v", filePath, err) } jsonData, err := json.MarshalIndent(packages, "", " ") if err != nil { log.Fatalf("Error marshalling to JSON: %v", err) } fmt.Println(string(jsonData)) } ``` And here is example output: ```json [ { "filename": "/path/to/appdirs/dist/appdirs-1.4.4-py2.py3-none-any.whl", "base_filename": "appdirs-1.4.4-py2.py3-none-any.whl", "metadata": { "metadata_version": "2.1", "name": "appdirs", "version": "1.4.4", "platforms": null, "supported_platforms": null, "summary": "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\".", "description": "\n.. image:: https://secure.travis-ci.org/ActiveState/appdirs.png\n :target: https://travis-ci.org/ActiveState/appdirs\n\nNote: This project has been officially deprecated. You may want to check out https://pypi.org/project/platformdirs/ which is a more active fork of appdirs. Thanks to everyone who has used appdirs. Shout out to ActiveState for the time they gave their employees to work on this over the years.\n\nthe problem\n===========\n\nWhat directory should your app use for storing user data? If running on macOS, you\nshould use::\n\n ~/Library/Application Support/\u003cAppName\u003e\n\nIf on Windows (at least English Win XP) that should be::\n\n C:\\Documents and Settings\\\u003cUser\u003e\\Application Data\\Local Settings\\\u003cAppAuthor\u003e\\\u003cAppName\u003e\n\nor possibly::\n\n C:\\Documents and Settings\\\u003cUser\u003e\\Application Data\\\u003cAppAuthor\u003e\\\u003cAppName\u003e\n\nfor `roaming profiles \u003chttps://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-vista/cc766489(v=ws.10)\u003e`_ but that is another story.\n\nOn Linux (and other Unices) the dir, according to the `XDG\nspec \u003chttps://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html\u003e`_, is::\n\n ~/.local/share/\u003cAppName\u003e\n\n\n``appdirs`` to the rescue\n=========================\n\nThis kind of thing is what the ``appdirs`` module is for. ``appdirs`` will\nhelp you choose an appropriate:\n\n- user data dir (``user_data_dir``)\n- user config dir (``user_config_dir``)\n- user cache dir (``user_cache_dir``)\n- site data dir (``site_data_dir``)\n- site config dir (``site_config_dir``)\n- user log dir (``user_log_dir``)\n\nand also:\n\n- is a single module so other Python packages can include their own private copy\n- is slightly opinionated on the directory names used. Look for \"OPINION\" in\n documentation and code for when an opinion is being applied.\n\n\nsome example output\n===================\n\nOn macOS::\n\n \u003e\u003e\u003e from appdirs import *\n \u003e\u003e\u003e appname = \"SuperApp\"\n \u003e\u003e\u003e appauthor = \"Acme\"\n \u003e\u003e\u003e user_data_dir(appname, appauthor)\n '/Users/trentm/Library/Application Support/SuperApp'\n \u003e\u003e\u003e site_data_dir(appname, appauthor)\n '/Library/Application Support/SuperApp'\n \u003e\u003e\u003e user_cache_dir(appname, appauthor)\n '/Users/trentm/Library/Caches/SuperApp'\n \u003e\u003e\u003e user_log_dir(appname, appauthor)\n '/Users/trentm/Library/Logs/SuperApp'\n\nOn Windows 7::\n\n \u003e\u003e\u003e from appdirs import *\n \u003e\u003e\u003e appname = \"SuperApp\"\n \u003e\u003e\u003e appauthor = \"Acme\"\n \u003e\u003e\u003e user_data_dir(appname, appauthor)\n 'C:\\\\Users\\\\trentm\\\\AppData\\\\Local\\\\Acme\\\\SuperApp'\n \u003e\u003e\u003e user_data_dir(appname, appauthor, roaming=True)\n 'C:\\\\Users\\\\trentm\\\\AppData\\\\Roaming\\\\Acme\\\\SuperApp'\n \u003e\u003e\u003e user_cache_dir(appname, appauthor)\n 'C:\\\\Users\\\\trentm\\\\AppData\\\\Local\\\\Acme\\\\SuperApp\\\\Cache'\n \u003e\u003e\u003e user_log_dir(appname, appauthor)\n 'C:\\\\Users\\\\trentm\\\\AppData\\\\Local\\\\Acme\\\\SuperApp\\\\Logs'\n\nOn Linux::\n\n \u003e\u003e\u003e from appdirs import *\n \u003e\u003e\u003e appname = \"SuperApp\"\n \u003e\u003e\u003e appauthor = \"Acme\"\n \u003e\u003e\u003e user_data_dir(appname, appauthor)\n '/home/trentm/.local/share/SuperApp\n \u003e\u003e\u003e site_data_dir(appname, appauthor)\n '/usr/local/share/SuperApp'\n \u003e\u003e\u003e site_data_dir(appname, appauthor, multipath=True)\n '/usr/local/share/SuperApp:/usr/share/SuperApp'\n \u003e\u003e\u003e user_cache_dir(appname, appauthor)\n '/home/trentm/.cache/SuperApp'\n \u003e\u003e\u003e user_log_dir(appname, appauthor)\n '/home/trentm/.cache/SuperApp/log'\n \u003e\u003e\u003e user_config_dir(appname)\n '/home/trentm/.config/SuperApp'\n \u003e\u003e\u003e site_config_dir(appname)\n '/etc/xdg/SuperApp'\n \u003e\u003e\u003e os.environ['XDG_CONFIG_DIRS'] = '/etc:/usr/local/etc'\n \u003e\u003e\u003e site_config_dir(appname, multipath=True)\n '/etc/SuperApp:/usr/local/etc/SuperApp'\n\n\n``AppDirs`` for convenience\n===========================\n\n::\n\n \u003e\u003e\u003e from appdirs import AppDirs\n \u003e\u003e\u003e dirs = AppDirs(\"SuperApp\", \"Acme\")\n \u003e\u003e\u003e dirs.user_data_dir\n '/Users/trentm/Library/Application Support/SuperApp'\n \u003e\u003e\u003e dirs.site_data_dir\n '/Library/Application Support/SuperApp'\n \u003e\u003e\u003e dirs.user_cache_dir\n '/Users/trentm/Library/Caches/SuperApp'\n \u003e\u003e\u003e dirs.user_log_dir\n '/Users/trentm/Library/Logs/SuperApp'\n\n\n \nPer-version isolation\n=====================\n\nIf you have multiple versions of your app in use that you want to be\nable to run side-by-side, then you may want version-isolation for these\ndirs::\n\n \u003e\u003e\u003e from appdirs import AppDirs\n \u003e\u003e\u003e dirs = AppDirs(\"SuperApp\", \"Acme\", version=\"1.0\")\n \u003e\u003e\u003e dirs.user_data_dir\n '/Users/trentm/Library/Application Support/SuperApp/1.0'\n \u003e\u003e\u003e dirs.site_data_dir\n '/Library/Application Support/SuperApp/1.0'\n \u003e\u003e\u003e dirs.user_cache_dir\n '/Users/trentm/Library/Caches/SuperApp/1.0'\n \u003e\u003e\u003e dirs.user_log_dir\n '/Users/trentm/Library/Logs/SuperApp/1.0'\n\n\n\nappdirs Changelog\n=================\n\nUNRELEASED\n----------\n- Add Python 3.7 support\n- Remove support for end-of-life Pythons 2.6, 3.2, and 3.3\n\nappdirs 1.4.4\n-------------\n- [PR #92] Don't import appdirs from setup.py which resolves issue #91\n- [PR #100] Corrects the config directory on OSX/macOS, which resolves issue #63.\n\nProject officially classified as Stable which is important\nfor inclusion in other distros such as ActivePython.\n\nappdirs 1.4.3\n-------------\n- [PR #76] Python 3.6 invalid escape sequence deprecation fixes\n- Fix for Python 3.6 support\n\nappdirs 1.4.2\n-------------\n- [PR #84] Allow installing without setuptools\n- [PR #86] Fix string delimiters in setup.py description\n- Add Python 3.6 support\n\nappdirs 1.4.1\n-------------\n- [issue #38] Fix _winreg import on Windows Py3\n- [issue #55] Make appname optional\n\nappdirs 1.4.0\n-------------\n- [PR #42] AppAuthor is now optional on Windows\n- [issue 41] Support Jython on Windows, Mac, and Unix-like platforms. Windows\n support requires `JNA \u003chttps://github.com/twall/jna\u003e`_.\n- [PR #44] Fix incorrect behaviour of the site_config_dir method\n\nappdirs 1.3.0\n-------------\n- [Unix, issue 16] Conform to XDG standard, instead of breaking it for\n everybody\n- [Unix] Removes gratuitous case mangling of the case, since \\*nix-es are\n usually case sensitive, so mangling is not wise\n- [Unix] Fixes the utterly wrong behaviour in ``site_data_dir``, return result\n based on XDG_DATA_DIRS and make room for respecting the standard which\n specifies XDG_DATA_DIRS is a multiple-value variable\n- [Issue 6] Add ``*_config_dir`` which are distinct on nix-es, according to\n XDG specs; on Windows and Mac return the corresponding ``*_data_dir``\n\nappdirs 1.2.0\n-------------\n\n- [Unix] Put ``user_log_dir`` under the *cache* dir on Unix. Seems to be more\n typical.\n- [issue 9] Make ``unicode`` work on py3k.\n\nappdirs 1.1.0\n-------------\n\n- [issue 4] Add ``AppDirs.user_log_dir``.\n- [Unix, issue 2, issue 7] appdirs now conforms to `XDG base directory spec\n \u003chttps://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html\u003e`_.\n- [Mac, issue 5] Fix ``site_data_dir()`` on Mac.\n- [Mac] Drop use of 'Carbon' module in favour of hardcoded paths; supports\n Python3 now.\n- [Windows] Append \"Cache\" to ``user_cache_dir`` on Windows by default. Use\n ``opinion=False`` option to disable this.\n- Add ``appdirs.AppDirs`` convenience class. Usage:\n\n \u003e\u003e\u003e dirs = AppDirs(\"SuperApp\", \"Acme\", version=\"1.0\")\n \u003e\u003e\u003e dirs.user_data_dir\n '/Users/trentm/Library/Application Support/SuperApp/1.0'\n\n- [Windows] Cherry-pick Komodo's change to downgrade paths to the Windows short\n paths if there are high bit chars.\n- [Linux] Change default ``user_cache_dir()`` on Linux to be singular, e.g.\n \"~/.superapp/cache\".\n- [Windows] Add ``roaming`` option to ``user_data_dir()`` (for use on Windows only)\n and change the default ``user_data_dir`` behaviour to use a *non*-roaming\n profile dir (``CSIDL_LOCAL_APPDATA`` instead of ``CSIDL_APPDATA``). Why? Because\n a large roaming profile can cause login speed issues. The \"only syncs on\n logout\" behaviour can cause surprises in appdata info.\n\n\nappdirs 1.0.1 (never released)\n------------------------------\n\nStarted this changelog 27 July 2010. Before that this module originated in the\n`Komodo \u003chttps://www.activestate.com/komodo-ide\u003e`_ product as ``applib.py`` and then\nas `applib/location.py\n\u003chttps://github.com/ActiveState/applib/blob/master/applib/location.py\u003e`_ (used by\n`PyPM \u003chttps://code.activestate.com/pypm/\u003e`_ in `ActivePython\n\u003chttps://www.activestate.com/activepython\u003e`_). This is basically a fork of\napplib.py 1.0.1 and applib/location.py 1.0.1.\n", "keywords": "application directory log cache user", "home_page": "https://github.com/ActiveState/appdirs", "download_url": "", "author": "Trent Mick", "author_email": "trentm@gmail.com", "license": "MIT", "classifiers": [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Libraries :: Python Modules" ], "requires": null, "provides": null, "obsoletes": null, "maintainer": "Jeff Rouse", "maintainer_email": "jr@its.to", "requires_python": "\u003e=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", "requires_external": null, "requires_dist": null, "provides_dist": null, "obsoletes_dist": null, "project_urls": null, "provides_extras": null, "description_content_type": "", "dynamic": null, "filename": "/path/to/appdirs/dist/appdirs-1.4.4-py2.py3-none-any.whl", "base_filename": "appdirs-1.4.4-py2.py3-none-any.whl" }, "python_version": "py2.py3", "file_type": "bdist_wheel", "safe_name": "appdirs", "signed_filename": "/path/to/appdirs/dist/appdirs-1.4.4-py2.py3-none-any.whl.asc", "signed_base_filename": "appdirs-1.4.4-py2.py3-none-any.whl.asc", "gpg_signature": null, "md5_digest": "3960d7797a1fb32d1229c59db8702c11", "sha2_digest": "2cf70fb91a350c7987e3410693978417056e5479aa476c0f4c68b3697d431776", "blake2_256_digest": "2a8c9ca4a072d43e00503f16d08db68d690644949ac3e4704ba6c2e7028e8402" } ] ```python-distribution-parser-0.2.3/archiver/000077500000000000000000000000001451454526500206625ustar00rootroot00000000000000python-distribution-parser-0.2.3/archiver/archive_reader.go000066400000000000000000000055531451454526500241640ustar00rootroot00000000000000package archiver import ( "archive/tar" "archive/zip" "compress/gzip" "errors" "fmt" "io" "os" "strings" ) // archiveReader is an interface to abstract the behavior of different archive types. type archiveReader interface { FileNames() ([]string, error) ReadFile(name string) ([]byte, error) Close() error } type zipReader struct { *zip.ReadCloser } func (z *zipReader) FileNames() ([]string, error) { var names []string for _, f := range z.File { names = append(names, f.Name) } return names, nil } func (z *zipReader) ReadFile(name string) ([]byte, error) { for _, f := range z.File { if f.Name == name { rc, err := f.Open() if err != nil { return nil, err } defer rc.Close() return io.ReadAll(rc) } } return nil, fmt.Errorf("file not found: %s", name) } type tarReader struct { filename string *tar.Reader closer io.Closer } func (t *tarReader) resetReader() error { t.Close() // Reopen the file f, err := os.Open(t.filename) if err != nil { return err } if strings.HasSuffix(t.filename, ".tar.gz") || strings.HasSuffix(t.filename, ".tgz") { gzr, err := gzip.NewReader(f) if err != nil { f.Close() return err } t.Reader = tar.NewReader(gzr) // Reset the tar reader with new gzip reader } return nil } func (t *tarReader) FileNames() ([]string, error) { defer t.resetReader() var names []string for { hdr, err := t.Next() if err == io.EOF { break } if err != nil { return nil, err } names = append(names, hdr.Name) } return names, nil } func (t *tarReader) ReadFile(name string) ([]byte, error) { for { hdr, err := t.Next() if err == io.EOF { return nil, fmt.Errorf("file not found: %s", name) } if err != nil { return nil, err } if hdr.Name == name { return io.ReadAll(t) } } } func (t *tarReader) Close() error { return t.closer.Close() } func NewArchiveReader(fqn string) (archiveReader, error) { _, err := os.Stat(fqn) if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("no such file: %s", fqn) } else if err != nil { return nil, err } if r, err := zip.OpenReader(fqn); err == nil { return &zipReader{r}, nil } f, err := os.Open(fqn) if err != nil { return nil, err } // check if file is a gzipped tarball if strings.HasSuffix(fqn, ".tar.gz") || strings.HasSuffix(fqn, ".tgz") { gzr, err := gzip.NewReader(f) if err != nil { f.Close() return nil, err } r := tar.NewReader(gzr) err = tarReadCheck(r) if err != nil { return nil, err } return &tarReader{fqn, r, f}, nil } f.Close() return nil, fmt.Errorf("not a known archive format: %s", fqn) } // tarReadCheck will attempt to read a header from the tar.Reader to // determine if it is a valid tar file. It will return nil if it can read // a header successfully; otherwise, it will return an error. func tarReadCheck(r *tar.Reader) error { _, err := r.Next() return err } python-distribution-parser-0.2.3/distributions/000077500000000000000000000000001451454526500217615ustar00rootroot00000000000000python-distribution-parser-0.2.3/distributions/distribution.go000066400000000000000000000201531451454526500250300ustar00rootroot00000000000000package distributions import ( "fmt" "io" "net/mail" "reflect" "strings" "unicode" "golang.org/x/text/encoding/charmap" ) func parse(r io.Reader) (*mail.Message, error) { return mail.ReadMessage(r) } func getHeaderValue(data *mail.Message, header string) string { return collapseLeadingWS(header, data.Header.Get(header)) } func getAllHeaderValues(data *mail.Message, header string) []string { values := data.Header[header] headers := make([]string, 0) for _, value := range values { headers = append(headers, collapseLeadingWS(header, value)) } return headers } func mustDecode(value interface{}) (string, error) { switch v := value.(type) { case []byte: // First try to decode as utf-8 utfStr := string(v) for _, runeValue := range utfStr { if runeValue == 0xfffd { // unicode replacement character // if utf-8 fails, try decoding as latin1 latin1Decoder := charmap.ISO8859_1.NewDecoder() decoded, err := latin1Decoder.String(utfStr) if err != nil { return "", err } return decoded, nil } } return utfStr, nil case string: return v, nil default: return "", fmt.Errorf("unsupported type: %T", v) } } func collapseLeadingWS(header, txt string) string { if strings.ToLower(header) == "description" { // preserve newlines lines := strings.Split(strings.TrimSpace(txt), "\n") for i, line := range lines { if strings.HasPrefix(line, " ") { // 8 spaces lines[i] = line[8:] } } return strings.Join(lines, "\n") } else { lines := strings.FieldsFunc(txt, func(r rune) bool { return unicode.IsSpace(r) && r != '\n' }) return strings.Join(lines, " ") } } type HeaderAttr struct { HeaderName string AttrName string Multiple bool } var HeaderAttrs1_0 = []HeaderAttr{ // PEP 241 {"Metadata-Version", "metadata_version", false}, {"Name", "name", false}, {"Version", "version", false}, {"Platform", "platforms", true}, {"Supported-Platform", "supported_platforms", true}, {"Summary", "summary", false}, {"Description", "description", false}, {"Keywords", "keywords", false}, {"Home-Page", "home_page", false}, {"Author", "author", false}, {"Author-Email", "author_email", false}, {"License", "license", false}, } var HeaderAttrs1_1 = append(HeaderAttrs1_0, []HeaderAttr{ // PEP 314 {"Classifier", "classifiers", true}, {"Download-URL", "download_url", false}, {"Requires", "requires", true}, {"Provides", "provides", true}, {"Obsoletes", "obsoletes", true}, }...) var HeaderAttrs1_2 = append(HeaderAttrs1_1, []HeaderAttr{ // PEP 345 {"Maintainer", "maintainer", false}, {"Maintainer-Email", "maintainer_email", false}, {"Requires-Python", "requires_python", false}, {"Requires-External", "requires_external", true}, {"Requires-Dist", "requires_dist", true}, {"Provides-Dist", "provides_dist", true}, {"Obsoletes-Dist", "obsoletes_dist", true}, {"Project-URL", "project_urls", true}, }...) var HeaderAttrs2_0 = HeaderAttrs1_2 //XXX PEP 426? var HeaderAttrs2_1 = append(HeaderAttrs1_2, []HeaderAttr{ // PEP 566 {"Provides-Extra", "provides_extras", true}, {"Description-Content-Type", "description_content_type", false}, }...) var HeaderAttrs2_2 = append(HeaderAttrs2_1, HeaderAttr{"Dynamic", "dynamic", true}) // PEP 643 var HeaderAttrs = map[string][]HeaderAttr{ "1.0": HeaderAttrs1_0, "1.1": HeaderAttrs1_1, "1.2": HeaderAttrs1_2, "2.0": HeaderAttrs2_0, "2.1": HeaderAttrs2_1, "2.2": HeaderAttrs2_2, } type Distribution interface { ExtractMetadata() error Parse(data []byte) error // Helper method to get values GetName() string GetVersion() string GetPythonVersion() string // This is used to return a map of all the metadata, // similar to how twine passes the metadata in a multipart // form request. MetadataMap() map[string][]string } type BaseDistribution struct { MetadataVersion string `json:"metadata_version"` // version 1.0 Name string `json:"name"` Version string `json:"version"` Platforms []string `json:"platforms"` SupportedPlatforms []string `json:"supported_platforms"` Summary string `json:"summary"` Description string `json:"description"` Keywords string `json:"keywords"` HomePage string `json:"home_page"` DownloadURL string `json:"download_url"` Author string `json:"author"` AuthorEmail string `json:"author_email"` License string `json:"license"` // version 1.1 Classifiers []string `json:"classifiers"` Requires []string `json:"requires"` Provides []string `json:"provides"` Obsoletes []string `json:"obsoletes"` // version 1.2 Maintainer string `json:"maintainer"` MaintainerEmail string `json:"maintainer_email"` RequiresPython string `json:"requires_python"` RequiresExternal []string `json:"requires_external"` RequiresDist []string `json:"requires_dist"` ProvidesDist []string `json:"provides_dist"` ObsoletesDist []string `json:"obsoletes_dist"` ProjectURLs []string `json:"project_urls"` // version 2.1 ProvidesExtras []string `json:"provides_extras"` DescriptionContentType string `json:"description_content_type"` // version 2.2 Dynamic []string `json:"dynamic"` } func (bd *BaseDistribution) GetHeaderAttrs() []HeaderAttr { ha, exists := HeaderAttrs[bd.MetadataVersion] if exists { return ha } return []HeaderAttr{} } func (bd *BaseDistribution) Parse(data []byte) error { decodedData, err := mustDecode(data) if err != nil { return err } msg, err := parse(strings.NewReader(decodedData)) if err != nil { return err } headerValue := getHeaderValue(msg, "Metadata-Version") if bd.MetadataVersion == "" && headerValue != "" { bd.MetadataVersion = headerValue } for _, headerAttr := range bd.GetHeaderAttrs() { if headerAttr.AttrName == "metadata_version" { continue } headerValues := getAllHeaderValues(msg, headerAttr.HeaderName) if len(headerValues) != 0 { if headerAttr.Multiple { bd.setJSONValue(headerAttr.AttrName, headerValues) } else if headerValues[0] != "UNKNOWN" { bd.setJSONValue(headerAttr.AttrName, headerValues[0]) } } } body, err := io.ReadAll(msg.Body) if err != nil { return fmt.Errorf("error reading body: %v\n", err) } if body != nil { bd.setJSONValue("description", string(body)) } return nil } func (bd *BaseDistribution) GetName() string { return bd.Name } func (bd *BaseDistribution) GetVersion() string { return bd.Version } // Remember to implement this for other distributions // if they need something more specific (e.g. Wheels) func (bd *BaseDistribution) GetPythonVersion() string { return "" } func (bd *BaseDistribution) setJSONValue(fieldName string, value interface{}) error { v := reflect.ValueOf(bd).Elem() for i := 0; i < v.NumField(); i++ { fieldInfo := v.Type().Field(i) jsonTag := fieldInfo.Tag.Get("json") if jsonTag == fieldName { fieldVal := v.Field(i) if !fieldVal.CanSet() { return fmt.Errorf("cannot set field %s", fieldName) } fieldType := fieldVal.Type() val := reflect.ValueOf(value) if val.Type().ConvertibleTo(fieldType) { fieldVal.Set(val.Convert(fieldType)) } else { return fmt.Errorf("provided value type didn't match object field type") } return nil } } return fmt.Errorf("no such json field: %s in object", fieldName) } func StructToMap(input interface{}) map[string][]string { result := make(map[string][]string) v := reflect.ValueOf(input) if v.Kind() == reflect.Ptr { v = v.Elem() } t := reflect.TypeOf(input) if t.Kind() == reflect.Ptr { t = t.Elem() } for i := 0; i < v.NumField(); i++ { field := t.Field(i) fieldName := v.Type().Field(i).Name fieldValue := v.Field(i) jsonTag := field.Tag.Get("json") if jsonTag != "" { fieldName = jsonTag } if fieldValue.Kind() == reflect.Struct { subMap := StructToMap(fieldValue.Interface()) for subKey, subValue := range subMap { result[subKey] = subValue } } else { switch val := fieldValue.Interface().(type) { case string: result[fieldName] = []string{val} case []string: result[fieldName] = val default: result[fieldName] = []string{fmt.Sprintf("%v", val)} } } } return result } python-distribution-parser-0.2.3/distributions/metadata.go000066400000000000000000000043661451454526500241010ustar00rootroot00000000000000package distributions import ( "errors" "fmt" "path/filepath" "strings" ) var DistExtensions = map[string]string{ ".whl": "bdist_wheel", ".tar.bz2": "sdist", ".tar.gz": "sdist", ".zip": "sdist", // TODO maybe eventually add support for other distribution extensions // ".exe": "bdist_wininst", // ".egg": "bdist_egg", } func getFullExtension(filename string) string { parts := strings.Split(filename, ".") if len(parts) < 2 { return "" } // Check for tar.gz, tar.bz2, etc. if parts[len(parts)-1] == "gz" || parts[len(parts)-1] == "bz2" { if len(parts) < 3 { return "." + parts[len(parts)-1] } return "." + parts[len(parts)-2] + "." + parts[len(parts)-1] } return "." + parts[len(parts)-1] } func NewDistributionMetadata(filename string) (Distribution, string, string, error) { var metadata Distribution var fileType string for ext, dt := range DistExtensions { if getFullExtension(filename) == ext { fileType = dt break } } if fileType == "" { return nil, "", "", errors.New("unknown distribution format: " + filepath.Base(filename)) } var err error switch fileType { case "bdist_wheel": metadata, err = NewWheel(filename) case "sdist": metadata, err = NewSDist(filename) default: return nil, "", "", errors.New("invalid distribution type: " + fileType) // TODO maybe eventually add support for other distribution extensions // "bdist_wininst": WinInst, // "bdist_egg": BDist, } if err != nil { return nil, "", "", fmt.Errorf("invalid distribution file: %s, err: %v", filepath.Base(filename), err) } // If this encounters a metadata version it doesn't support, it may give us // back empty metadata. At the very least, we should have a name and version, // which could also be empty if, for example, a MANIFEST.in doesn't include // setup.cfg. if metadata.GetName() == "" || metadata.GetVersion() == "" { return nil, "", "", errors.New("metadata is missing required fields") } var pythonVersion string switch fileType { // TODO maybe eventually add support for other distribution extensions // case "bdist_egg": case "bdist_wheel", "bdist_wininst": pythonVersion = metadata.GetPythonVersion() default: pythonVersion = "" } return metadata, pythonVersion, fileType, nil } python-distribution-parser-0.2.3/distributions/sdist.go000066400000000000000000000033611451454526500234410ustar00rootroot00000000000000package distributions import ( "bytes" "fmt" "path/filepath" "sort" "strings" "github.com/rstudio/python-distribution-parser/archiver" ) type SDist struct { BaseDistribution Filename string `json:"file_name"` } func NewSDist(filename string) (Distribution, error) { sdist := &SDist{} sdist.Filename = filename err := sdist.ExtractMetadata() if err != nil { return nil, err } return sdist, nil } func (sd *SDist) MetadataMap() map[string][]string { return StructToMap(*sd) } func (sd *SDist) ExtractMetadata() error { data, err := sd.read() if err != nil { return err } err = sd.Parse(data) if err != nil { return err } return nil } func (sd *SDist) read() ([]byte, error) { filename := sd.Filename fqn, err := filepath.Abs(filepath.Clean(filename)) if err != nil { return nil, fmt.Errorf("error normalizing path: %w", err) } archiveReader, err := archiver.NewArchiveReader(fqn) if err != nil { return nil, fmt.Errorf("error getting archive: %w", err) } defer archiveReader.Close() // Ensure the archive is closed after reading fileNames, err := archiveReader.FileNames() if err != nil { return nil, err } var tuples [][]string for _, name := range fileNames { if strings.Contains(name, "PKG-INFO") { tuples = append(tuples, strings.Split(name, "/")) } } sort.Slice(tuples, func(i, j int) bool { return len(tuples[i]) < len(tuples[j]) }) for _, path := range tuples { candidate := strings.Join(path, "/") data, err := archiveReader.ReadFile(candidate) if err != nil { return nil, fmt.Errorf("error reading file %s from archive: %v", candidate, err) } if bytes.Contains(data, []byte("Metadata-Version")) { return data, nil } } return nil, fmt.Errorf("no PKG-INFO in archive: %s", fqn) } python-distribution-parser-0.2.3/distributions/wheel.go000066400000000000000000000044331451454526500234200ustar00rootroot00000000000000package distributions import ( "bytes" "fmt" "path/filepath" "regexp" "sort" "strings" "github.com/rstudio/python-distribution-parser/archiver" ) var wheelFileRe = regexp.MustCompile(`^(?P(?P.+?)(-(?P\d.+?))?)(?:(-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?)\.whl|\.dist-info)$`) type Wheel struct { BaseDistribution Filename string `json:"file_name"` BaseFilename string `json:"base_filename"` } func NewWheel(filename string) (Distribution, error) { wheel := &Wheel{} wheel.Filename = filename wheel.BaseFilename = filepath.Base(filename) err := wheel.ExtractMetadata() if err != nil { return nil, err } return wheel, nil } func (whl *Wheel) MetadataMap() map[string][]string { return StructToMap(*whl) } func (whl *Wheel) ExtractMetadata() error { data, err := whl.read() if err != nil { return err } err = whl.Parse(data) if err != nil { return err } return nil } func (whl *Wheel) read() ([]byte, error) { filename := whl.Filename fqn, err := filepath.Abs(filepath.Clean(filename)) if err != nil { return nil, fmt.Errorf("error normalizing path: %w", err) } archiveReader, err := archiver.NewArchiveReader(fqn) if err != nil { return nil, fmt.Errorf("error getting archive: %w", err) } defer archiveReader.Close() // Ensure the archive is closed after reading fileNames, err := archiveReader.FileNames() if err != nil { return nil, err } var tuples [][]string for _, name := range fileNames { if strings.Contains(name, "METADATA") { tuples = append(tuples, strings.Split(name, "/")) } } sort.Slice(tuples, func(i, j int) bool { return len(tuples[i]) < len(tuples[j]) }) for _, path := range tuples { candidate := strings.Join(path, "/") data, err := archiveReader.ReadFile(candidate) if err != nil { return nil, fmt.Errorf("error reading file %s from archive: %v", candidate, err) } if bytes.Contains(data, []byte("Metadata-Version")) { return data, nil } } return nil, fmt.Errorf("no METADATA in archive: %s", fqn) } func (whl *Wheel) GetPythonVersion() string { wheelInfo := wheelFileRe.FindStringSubmatch(whl.BaseFilename) if wheelInfo == nil { return "any" } for i, name := range wheelFileRe.SubexpNames() { if name == "pyver" { return wheelInfo[i] } } return "any" } python-distribution-parser-0.2.3/go.mod000066400000000000000000000002611451454526500201640ustar00rootroot00000000000000module github.com/rstudio/python-distribution-parser go 1.21 require ( golang.org/x/crypto v0.13.0 golang.org/x/text v0.13.0 ) require golang.org/x/sys v0.12.0 // indirect python-distribution-parser-0.2.3/go.sum000066400000000000000000000007231451454526500202140ustar00rootroot00000000000000golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= python-distribution-parser-0.2.3/packages/000077500000000000000000000000001451454526500206355ustar00rootroot00000000000000python-distribution-parser-0.2.3/packages/hash_manager.go000066400000000000000000000042201451454526500235770ustar00rootroot00000000000000package packages import ( "crypto/md5" "crypto/sha256" "encoding/hex" "hash" "io" "os" "golang.org/x/crypto/blake2b" ) // Hexdigest struct to hold the hashes type Hexdigest struct { md5 string sha2 string blake2 string } // HashManager struct to manage hash values type HashManager struct { filename string md5 *md5Hasher sha2 sha256Hasher blake2 *blake2Hasher } type md5Hasher struct { hasher hash.Hash } type sha256Hasher struct { hasher hash.Hash } type blake2Hasher struct { hasher hash.Hash } func (h *md5Hasher) update(content []byte) { if h != nil { h.hasher.Write(content) } } func (h *md5Hasher) hexdigest() string { if h == nil { return "" } hash := h.hasher.Sum(nil) hashStr := hex.EncodeToString(hash) return hashStr } func (h *sha256Hasher) update(content []byte) { h.hasher.Write(content) } func (h *sha256Hasher) hexdigest() string { hash := h.hasher.Sum(nil) hashStr := hex.EncodeToString(hash) return hashStr } func (h *blake2Hasher) update(content []byte) { if h != nil { h.hasher.Write(content) } } func (h *blake2Hasher) hexdigest() string { if h == nil { return "" } hash := h.hasher.Sum(nil) hashStr := hex.EncodeToString(hash) return hashStr } // NewHashManager creates a new HashManager func NewHashManager(filename string) (*HashManager, error) { m := &md5Hasher{hasher: md5.New()} s := sha256Hasher{hasher: sha256.New()} blake2, err := blake2b.New256(nil) if err != nil { return nil, err } b := &blake2Hasher{hasher: blake2} return &HashManager{filename: filename, md5: m, sha2: s, blake2: b}, nil } func (hm *HashManager) Hash() error { file, err := os.Open(hm.filename) if err != nil { return err } defer file.Close() buffer := make([]byte, 64*1024) for { bytesRead, err := file.Read(buffer) if err != nil && err != io.EOF { return err } if bytesRead == 0 { break } content := buffer[:bytesRead] hm.md5.update(content) hm.sha2.update(content) hm.blake2.update(content) } return nil } func (hm *HashManager) HexDigest() *Hexdigest { return &Hexdigest{ md5: hm.md5.hexdigest(), sha2: hm.sha2.hexdigest(), blake2: hm.blake2.hexdigest(), } } python-distribution-parser-0.2.3/packages/package.go000066400000000000000000000066551451454526500225730ustar00rootroot00000000000000package packages import ( "errors" "io" "os" "path/filepath" "regexp" "github.com/rstudio/python-distribution-parser/distributions" ) type PackageFile struct { Filename string `json:"file_name"` Comment string `json:"comment"` BaseFilename string `json:"base_filename"` Metadata *distributions.Distribution `json:"metadata"` PythonVersion string `json:"pyversion"` FileType string `json:"filetype"` SafeName string `json:"safe_name"` SignedFilename string `json:"signed_filename"` SignedBaseFilename string `json:"signed_base_filename"` GPGSignature *Signature `json:"gpg_signature"` MD5Digest string `json:"md5_digest"` SHA2Digest string `json:"sha256_digest"` Blake2_256Digest string `json:"blake2_256_digest"` } type Signature struct { Filename string `json:"signed_filename"` Bytes []byte `json:"signed_bytes"` } // Convert an arbitrary string to a standard distribution name. // Any runs of non-alphanumeric/. characters are replaced with a single '-'. // Copied from pkg_resources.safe_name for compatibility with warehouse. // See https://github.com/pypa/twine/issues/743. func safeName(name string) string { reg := regexp.MustCompile("[^A-Za-z0-9.]+") return reg.ReplaceAllString(name, "-") } func NewPackageFile(filename string) (*PackageFile, error) { metadata, pythonVersion, fileType, err := distributions.NewDistributionMetadata(filename) if err != nil { return nil, err } baseFilename := filepath.Base(filename) safeName := safeName(metadata.GetName()) signedFilename := filename + ".asc" signedBaseFilename := baseFilename + ".asc" hashManager, err := NewHashManager(filename) if err != nil { return nil, err } hashManager.Hash() hexdigest := hashManager.HexDigest() return &PackageFile{ Filename: filename, Comment: "", // Adding a comment isn't currently possible BaseFilename: baseFilename, Metadata: &metadata, PythonVersion: pythonVersion, FileType: fileType, SafeName: safeName, SignedFilename: signedFilename, SignedBaseFilename: signedBaseFilename, MD5Digest: hexdigest.md5, SHA2Digest: hexdigest.sha2, Blake2_256Digest: hexdigest.blake2, }, nil } func (pf *PackageFile) MetadataMap() map[string][]string { pkgMap := distributions.StructToMap(pf) metadata := *pf.Metadata metadataMap := metadata.MetadataMap() result := make(map[string][]string, len(pkgMap)+len(metadataMap)) for pk, pv := range pkgMap { result[pk] = pv } for mk, mv := range metadataMap { result[mk] = mv } // This makes the request look more like Twine result["protocol_version"] = []string{"1"} delete(result, "metadata") return result } func (pf *PackageFile) AddGPGSignature(signatureFilepath string, signatureFilename string) error { if pf.GPGSignature != nil { return errors.New("GPG Signature can only be added once") } gpg, err := os.Open(signatureFilepath) if err != nil { return err } defer gpg.Close() bytes, err := io.ReadAll(gpg) if err != nil { return err } pf.GPGSignature = &Signature{ Filename: signatureFilename, Bytes: bytes, } return nil } python-distribution-parser-0.2.3/packages/utils.go000066400000000000000000000007751451454526500223350ustar00rootroot00000000000000package packages import ( "fmt" "os" ) // GetFileSize returns the size of a file in KB, or MB if >= 1024 KB. func GetFileSize(filename string) (string, error) { file, err := os.Stat(filename) if err != nil { return "", err } // convert file size to KB fileSize := float64(file.Size()) / 1024 sizeUnit := "KB" if fileSize > 1024 { // convert file size to MB if it's more than 1024 KB fileSize = fileSize / 1024 sizeUnit = "MB" } return fmt.Sprintf("%.1f %s", fileSize, sizeUnit), nil } python-distribution-parser-0.2.3/parse.go000066400000000000000000000062121451454526500205210ustar00rootroot00000000000000package parse import ( "errors" "fmt" "os" "path/filepath" "sort" "strings" "github.com/rstudio/python-distribution-parser/packages" ) func endsWith(str, suffix string) bool { return len(str) >= len(suffix) && str[len(str)-len(suffix):] == suffix } func groupWheelFilesFirst(files []string) []string { // Check if there's any wheel file hasWheel := false for _, fname := range files { if endsWith(fname, ".whl") { hasWheel = true break } } if !hasWheel { return files } sort.Slice(files, func(i, j int) bool { return endsWith(files[i], ".whl") && !endsWith(files[j], ".whl") }) return files } func findDistributions(dists []string) ([]string, error) { var packages []string for _, filename := range dists { if _, err := os.Stat(filename); err == nil { packages = append(packages, filename) continue } files, err := filepath.Glob(filename) if err != nil { return nil, err } if files == nil || len(files) == 0 { return nil, fmt.Errorf("cannot find file (or expand pattern): %s", filename) } packages = append(packages, files...) } return groupWheelFilesFirst(packages), nil } func makePackage(filename string, signatures map[string]string) (*packages.PackageFile, error) { packageFile, err := packages.NewPackageFile(filename) if err != nil { return nil, fmt.Errorf("unable to create packageFile from %s: %v\n", filename, err) } signedName := packageFile.SignedBaseFilename if signature, exists := signatures[signedName]; exists { packageFile.AddGPGSignature(signature, signedName) } _, err = packages.GetFileSize(packageFile.Filename) if err != nil { return nil, fmt.Errorf("%s is not a real file\n", packageFile.Filename) } return packageFile, nil } func parse(dists []string) ([]*packages.PackageFile, error) { dists, err := findDistributions(dists) if err != nil { return nil, fmt.Errorf("error finding distributions: %v\n", err) } // Initialize maps for signatures and a slice for distributions signatures := make(map[string]string) var distributions []string // Separate signatures and distributions for _, d := range dists { base := filepath.Base(d) if strings.HasSuffix(d, ".asc") { signatures[base] = d } else { distributions = append(distributions, d) } } var packages []*packages.PackageFile for _, filename := range distributions { p, err := makePackage(filename, signatures) if err != nil { return nil, err } packages = append(packages, p) } return packages, nil } func Parse(path string) ([]*packages.PackageFile, error) { var files []string info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("%s does not exist: %v\n", path, err) } return nil, err } if info.IsDir() { dirFiles, err := os.ReadDir(path) if err != nil { return nil, err } for _, entry := range dirFiles { // Don't recursively go in directories, only go one level deep. if !entry.IsDir() { fullPath := filepath.Join(path, entry.Name()) files = append(files, fullPath) } } } else { files = append(files, path) } if len(files) == 0 { return nil, errors.New("no files to parse") } return parse(files) }