Source code for openapi_client.models.dataset_metadata
# coding: utf-8
"""
Amorphic Data Platform
Amorphic Data Platform - API Definition documentation
The version of the OpenAPI document: 0.3.0
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from openapi_client.models.advanced_config import AdvancedConfig
from openapi_client.models.column_name_and_type import ColumnNameAndType
from openapi_client.models.data_metrics_collection_options import DataMetricsCollectionOptions
from openapi_client.models.dataset_life_cycle_rules import DatasetLifeCycleRules
from openapi_client.models.key_schema import KeySchema
from openapi_client.models.malware_detection_options import MalwareDetectionOptions
from typing import Optional, Set
from typing_extensions import Self
[docs]
class DatasetMetadata(BaseModel):
"""
DatasetMetadata
""" # noqa: E501
dataset_name: StrictStr = Field(alias="DatasetName")
display_name: Optional[StrictStr] = Field(default=None, alias="DisplayName")
dataset_description: StrictStr = Field(alias="DatasetDescription")
data_classification: Optional[List[StrictStr]] = Field(default=None, alias="DataClassification")
domain: StrictStr = Field(alias="Domain")
dataset_type: StrictStr = Field(alias="DatasetType")
file_type: Optional[StrictStr] = Field(default=None, alias="FileType")
target_location: StrictStr = Field(alias="TargetLocation")
keywords: List[StrictStr] = Field(alias="Keywords")
table_update: Optional[StrictStr] = Field(default=None, alias="TableUpdate")
file_delimiter: Optional[StrictStr] = Field(default=None, alias="FileDelimiter")
skip_file_header: Optional[StrictBool] = Field(default=None, alias="SkipFileHeader")
skip_lz_process: Optional[StrictBool] = Field(default=None, alias="SkipLZProcess")
datasource_type: Optional[StrictStr] = Field(default=None, alias="DatasourceType")
table_name: Optional[StrictStr] = Field(default=None, alias="TableName")
directory_path: Optional[StrictStr] = Field(default=None, alias="DirectoryPath")
is_data_profiling_enabled: Optional[StrictBool] = Field(default=None, alias="IsDataProfilingEnabled")
is_data_cleanup_enabled: Optional[StrictBool] = Field(default=None, alias="IsDataCleanupEnabled")
malware_detection_options: Optional[MalwareDetectionOptions] = Field(default=None, alias="MalwareDetectionOptions")
target_table_prep_mode: Optional[StrictStr] = Field(default=None, alias="TargetTablePrepMode")
is_data_validation_enabled: Optional[StrictBool] = Field(default=None, alias="IsDataValidationEnabled")
are_ai_services_enabled: Optional[StrictBool] = Field(default=None, alias="AreAIServicesEnabled")
is_translate_ai_results_enabled: Optional[StrictBool] = Field(default=None, alias="IsTranslateAIResultsEnabled")
life_cycle_policy_status: Optional[StrictStr] = Field(default=None, alias="LifeCyclePolicyStatus")
life_cycle_rules: Optional[DatasetLifeCycleRules] = Field(default=None, alias="LifeCycleRules")
data_metrics_collection_options: Optional[DataMetricsCollectionOptions] = Field(default=None, alias="DataMetricsCollectionOptions")
advanced_config: Optional[AdvancedConfig] = Field(default=None, alias="AdvancedConfig")
datasource_id: Optional[StrictStr] = Field(default=None, alias="DatasourceId")
dataset_s3_path: Optional[StrictStr] = Field(default=None, alias="DatasetS3Path")
ser_de: Optional[StrictStr] = Field(default=None, alias="SerDe")
view_type: Optional[StrictStr] = Field(default=None, alias="ViewType")
sql_statement: Optional[StrictStr] = Field(default=None, alias="SqlStatement")
encoding: Optional[StrictStr] = Field(default=None, alias="Encoding")
assume_role: Optional[StrictStr] = Field(default=None, alias="AssumeRole")
is_delta_lake_table: Optional[StrictBool] = Field(default=None, alias="IsDeltaLakeTable")
skip_trash: Optional[StrictBool] = Field(default=None, alias="SkipTrash")
auto_refresh: Optional[StrictBool] = Field(default=None, alias="AutoRefresh")
data_cleanup_duration: Optional[StrictInt] = Field(default=None, alias="DataCleanupDuration")
update_schema: Optional[StrictStr] = Field(default=None, alias="UpdateSchema")
dataset_schema: Optional[List[ColumnNameAndType]] = Field(default=None, alias="DatasetSchema")
skip_row_count: Optional[Dict[str, Dict[str, Any]]] = Field(default=None, alias="SkipRowCount")
hudi_table_options: Optional[Dict[str, Dict[str, Any]]] = Field(default=None, alias="HudiTableOptions")
iceberg_table_options: Optional[Dict[str, Dict[str, Any]]] = Field(default=None, alias="IcebergTableOptions")
precombine_key: Optional[StrictStr] = Field(default=None, alias="PrecombineKey")
primary_key: Optional[StrictStr] = Field(default=None, alias="PrimaryKey")
storage_type: Optional[StrictStr] = Field(default=None, alias="StorageType")
input_regex: Optional[StrictStr] = Field(default=None, alias="InputRegex")
partition_keys: Optional[List[KeySchema]] = Field(default=None, alias="PartitionKeys")
global_secondary_indexes: Optional[List[KeySchema]] = Field(default=None, alias="GlobalSecondaryIndexes")
local_secondary_indexes: Optional[List[KeySchema]] = Field(default=None, alias="LocalSecondaryIndexes")
dist_key: Optional[StrictStr] = Field(default=None, alias="DistKey")
dist_type: Optional[StrictStr] = Field(default=None, alias="DistType")
is_identity_needed: Optional[StrictStr] = Field(default=None, alias="IsIdentityNeeded")
sort_keys: Optional[StrictStr] = Field(default=None, alias="SortKeys")
sort_type: Optional[StrictStr] = Field(default=None, alias="SortType")
record_keys: Optional[List[StrictStr]] = Field(default=None, alias="RecordKeys")
pagination_type: Optional[StrictStr] = Field(default=None, alias="PaginationType")
pagination_header_key: Optional[StrictStr] = Field(default=None, alias="PaginationHeaderKey")
catalog_id: Optional[StrictStr] = Field(default=None, alias="CatalogId")
cors_dataset: Optional[StrictStr] = Field(default=None, alias="CORSDataset")
partition_key: Optional[KeySchema] = Field(default=None, alias="PartitionKey")
sort_key: Optional[KeySchema] = Field(default=None, alias="SortKey")
primary_keys: Optional[List[StrictStr]] = Field(default=None, alias="PrimaryKeys")
unique_columns: Optional[List[StrictStr]] = Field(default=None, alias="UniqueColumns")
not_null_columns: Optional[List[StrictStr]] = Field(default=None, alias="NotNullColumns")
dq_constraints: Optional[List[Dict[str, Any]]] = Field(default=None, alias="DqConstraints")
are_auto_constraint_suggestions_enabled: Optional[StrictBool] = Field(default=None, alias="AreAutoConstraintSuggestionsEnabled")
data_conversion_params: Optional[Dict[str, Any]] = Field(default=None, alias="DataConversionParams")
column_table_constraints: Optional[Dict[str, Any]] = Field(default=None, alias="ColumnTableConstraints")
dataset_key_options: Optional[Dict[str, Any]] = Field(default=None, alias="DatasetKeyOptions")
latest_record_indicator: Optional[ColumnNameAndType] = Field(default=None, alias="LatestRecordIndicator")
__properties: ClassVar[List[str]] = ["DatasetName", "DisplayName", "DatasetDescription", "DataClassification", "Domain", "DatasetType", "FileType", "TargetLocation", "Keywords", "TableUpdate", "FileDelimiter", "SkipFileHeader", "SkipLZProcess", "DatasourceType", "TableName", "DirectoryPath", "IsDataProfilingEnabled", "IsDataCleanupEnabled", "MalwareDetectionOptions", "TargetTablePrepMode", "IsDataValidationEnabled", "AreAIServicesEnabled", "IsTranslateAIResultsEnabled", "LifeCyclePolicyStatus", "LifeCycleRules", "DataMetricsCollectionOptions", "AdvancedConfig", "DatasourceId", "DatasetS3Path", "SerDe", "ViewType", "SqlStatement", "Encoding", "AssumeRole", "IsDeltaLakeTable", "SkipTrash", "AutoRefresh", "DataCleanupDuration", "UpdateSchema", "DatasetSchema", "SkipRowCount", "HudiTableOptions", "IcebergTableOptions", "PrecombineKey", "PrimaryKey", "StorageType", "InputRegex", "PartitionKeys", "GlobalSecondaryIndexes", "LocalSecondaryIndexes", "DistKey", "DistType", "IsIdentityNeeded", "SortKeys", "SortType", "RecordKeys", "PaginationType", "PaginationHeaderKey", "CatalogId", "CORSDataset", "PartitionKey", "SortKey", "PrimaryKeys", "UniqueColumns", "NotNullColumns", "DqConstraints", "AreAutoConstraintSuggestionsEnabled", "DataConversionParams", "ColumnTableConstraints", "DatasetKeyOptions", "LatestRecordIndicator"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
[docs]
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
[docs]
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
[docs]
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of DatasetMetadata from a JSON string"""
return cls.from_dict(json.loads(json_str))
[docs]
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of malware_detection_options
if self.malware_detection_options:
_dict['MalwareDetectionOptions'] = self.malware_detection_options.to_dict()
# override the default output from pydantic by calling `to_dict()` of life_cycle_rules
if self.life_cycle_rules:
_dict['LifeCycleRules'] = self.life_cycle_rules.to_dict()
# override the default output from pydantic by calling `to_dict()` of data_metrics_collection_options
if self.data_metrics_collection_options:
_dict['DataMetricsCollectionOptions'] = self.data_metrics_collection_options.to_dict()
# override the default output from pydantic by calling `to_dict()` of advanced_config
if self.advanced_config:
_dict['AdvancedConfig'] = self.advanced_config.to_dict()
# override the default output from pydantic by calling `to_dict()` of each item in dataset_schema (list)
_items = []
if self.dataset_schema:
for _item_dataset_schema in self.dataset_schema:
if _item_dataset_schema:
_items.append(_item_dataset_schema.to_dict())
_dict['DatasetSchema'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in partition_keys (list)
_items = []
if self.partition_keys:
for _item_partition_keys in self.partition_keys:
if _item_partition_keys:
_items.append(_item_partition_keys.to_dict())
_dict['PartitionKeys'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in global_secondary_indexes (list)
_items = []
if self.global_secondary_indexes:
for _item_global_secondary_indexes in self.global_secondary_indexes:
if _item_global_secondary_indexes:
_items.append(_item_global_secondary_indexes.to_dict())
_dict['GlobalSecondaryIndexes'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in local_secondary_indexes (list)
_items = []
if self.local_secondary_indexes:
for _item_local_secondary_indexes in self.local_secondary_indexes:
if _item_local_secondary_indexes:
_items.append(_item_local_secondary_indexes.to_dict())
_dict['LocalSecondaryIndexes'] = _items
# override the default output from pydantic by calling `to_dict()` of partition_key
if self.partition_key:
_dict['PartitionKey'] = self.partition_key.to_dict()
# override the default output from pydantic by calling `to_dict()` of sort_key
if self.sort_key:
_dict['SortKey'] = self.sort_key.to_dict()
# override the default output from pydantic by calling `to_dict()` of latest_record_indicator
if self.latest_record_indicator:
_dict['LatestRecordIndicator'] = self.latest_record_indicator.to_dict()
return _dict
[docs]
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of DatasetMetadata from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"DatasetName": obj.get("DatasetName"),
"DisplayName": obj.get("DisplayName"),
"DatasetDescription": obj.get("DatasetDescription"),
"DataClassification": obj.get("DataClassification"),
"Domain": obj.get("Domain"),
"DatasetType": obj.get("DatasetType"),
"FileType": obj.get("FileType"),
"TargetLocation": obj.get("TargetLocation"),
"Keywords": obj.get("Keywords"),
"TableUpdate": obj.get("TableUpdate"),
"FileDelimiter": obj.get("FileDelimiter"),
"SkipFileHeader": obj.get("SkipFileHeader"),
"SkipLZProcess": obj.get("SkipLZProcess"),
"DatasourceType": obj.get("DatasourceType"),
"TableName": obj.get("TableName"),
"DirectoryPath": obj.get("DirectoryPath"),
"IsDataProfilingEnabled": obj.get("IsDataProfilingEnabled"),
"IsDataCleanupEnabled": obj.get("IsDataCleanupEnabled"),
"MalwareDetectionOptions": MalwareDetectionOptions.from_dict(obj["MalwareDetectionOptions"]) if obj.get("MalwareDetectionOptions") is not None else None,
"TargetTablePrepMode": obj.get("TargetTablePrepMode"),
"IsDataValidationEnabled": obj.get("IsDataValidationEnabled"),
"AreAIServicesEnabled": obj.get("AreAIServicesEnabled"),
"IsTranslateAIResultsEnabled": obj.get("IsTranslateAIResultsEnabled"),
"LifeCyclePolicyStatus": obj.get("LifeCyclePolicyStatus"),
"LifeCycleRules": DatasetLifeCycleRules.from_dict(obj["LifeCycleRules"]) if obj.get("LifeCycleRules") is not None else None,
"DataMetricsCollectionOptions": DataMetricsCollectionOptions.from_dict(obj["DataMetricsCollectionOptions"]) if obj.get("DataMetricsCollectionOptions") is not None else None,
"AdvancedConfig": AdvancedConfig.from_dict(obj["AdvancedConfig"]) if obj.get("AdvancedConfig") is not None else None,
"DatasourceId": obj.get("DatasourceId"),
"DatasetS3Path": obj.get("DatasetS3Path"),
"SerDe": obj.get("SerDe"),
"ViewType": obj.get("ViewType"),
"SqlStatement": obj.get("SqlStatement"),
"Encoding": obj.get("Encoding"),
"AssumeRole": obj.get("AssumeRole"),
"IsDeltaLakeTable": obj.get("IsDeltaLakeTable"),
"SkipTrash": obj.get("SkipTrash"),
"AutoRefresh": obj.get("AutoRefresh"),
"DataCleanupDuration": obj.get("DataCleanupDuration"),
"UpdateSchema": obj.get("UpdateSchema"),
"DatasetSchema": [ColumnNameAndType.from_dict(_item) for _item in obj["DatasetSchema"]] if obj.get("DatasetSchema") is not None else None,
"SkipRowCount": obj.get("SkipRowCount"),
"HudiTableOptions": obj.get("HudiTableOptions"),
"IcebergTableOptions": obj.get("IcebergTableOptions"),
"PrecombineKey": obj.get("PrecombineKey"),
"PrimaryKey": obj.get("PrimaryKey"),
"StorageType": obj.get("StorageType"),
"InputRegex": obj.get("InputRegex"),
"PartitionKeys": [KeySchema.from_dict(_item) for _item in obj["PartitionKeys"]] if obj.get("PartitionKeys") is not None else None,
"GlobalSecondaryIndexes": [KeySchema.from_dict(_item) for _item in obj["GlobalSecondaryIndexes"]] if obj.get("GlobalSecondaryIndexes") is not None else None,
"LocalSecondaryIndexes": [KeySchema.from_dict(_item) for _item in obj["LocalSecondaryIndexes"]] if obj.get("LocalSecondaryIndexes") is not None else None,
"DistKey": obj.get("DistKey"),
"DistType": obj.get("DistType"),
"IsIdentityNeeded": obj.get("IsIdentityNeeded"),
"SortKeys": obj.get("SortKeys"),
"SortType": obj.get("SortType"),
"RecordKeys": obj.get("RecordKeys"),
"PaginationType": obj.get("PaginationType"),
"PaginationHeaderKey": obj.get("PaginationHeaderKey"),
"CatalogId": obj.get("CatalogId"),
"CORSDataset": obj.get("CORSDataset"),
"PartitionKey": KeySchema.from_dict(obj["PartitionKey"]) if obj.get("PartitionKey") is not None else None,
"SortKey": KeySchema.from_dict(obj["SortKey"]) if obj.get("SortKey") is not None else None,
"PrimaryKeys": obj.get("PrimaryKeys"),
"UniqueColumns": obj.get("UniqueColumns"),
"NotNullColumns": obj.get("NotNullColumns"),
"DqConstraints": obj.get("DqConstraints"),
"AreAutoConstraintSuggestionsEnabled": obj.get("AreAutoConstraintSuggestionsEnabled"),
"DataConversionParams": obj.get("DataConversionParams"),
"ColumnTableConstraints": obj.get("ColumnTableConstraints"),
"DatasetKeyOptions": obj.get("DatasetKeyOptions"),
"LatestRecordIndicator": ColumnNameAndType.from_dict(obj["LatestRecordIndicator"]) if obj.get("LatestRecordIndicator") is not None else None
})
return _obj