napkin-backend-bigquery
Safe HaskellNone
LanguageGHC2024

Napkin.Run.BigQuery

Synopsis

Documentation

normalizeTableNames :: BQProjectId -> Set (Ref Table) -> Map (Ref Table) NormalizedTable #

bqGetAnnotations :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> GoogleEnv -> m TableAnnotations #

renderBigQuery :: TableWriteStrategy -> Text #

pattern MaxResults :: Word32 #

exec :: forall a (scopes :: [Symbol]) m. (GoogleRequest a, KnownScopes scopes, SatisfyScope (Scopes a) scopes, MonadIO m) => Env scopes -> a -> m (Rs a) #

exec404 :: forall a (scopes :: [Symbol]) m e. (GoogleRequest a, KnownScopes scopes, SatisfyScope (Scopes a) scopes, MonadIO m) => e -> e -> Env scopes -> a -> m e #

getTableInfo :: MonadNapkin BigQuery m => GoogleEnv -> BQProjectId -> Ref Table -> m Table #

listTables :: MonadNapkin BigQuery m => GoogleEnv -> BQProjectId -> Prefix -> m (Set ListedTable) #

extractNormalized' :: BQProjectId -> Ref Table -> Either Text NormalizedTable #

BigQuery Names generally speaking have only three sections, subdivided by ".". Names cannot include a "." internally. But INFORMATION_SCHEMA views (read-only, system-defined views that provide metadata information about BigQuery objects) will have a "." in the name to demarcate. So there is an individual view called INFORMATION_SCHEMA.TABLES, one called INFORMATION_SCHEMA.VIEWS etc. If we wish to run against INFORMATION_SCHEMA views, we need to be a little clever in parsing them, allowing for the possibility that in fact two individual name segments actually are part of one view name.

extractNormalized :: BQProjectId -> Ref Table -> Either Text NormalizedTable #

bqValidator :: (SpecDependency -> Ref Table) -> SpecsDepsTables -> [WithSpecTable Text] #

newtype Dollars #

Constructors

Dollars 

Fields

Instances

Instances details
ToJSON Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

toJSON :: Dollars -> Value #

toEncoding :: Dollars -> Encoding #

toJSONList :: [Dollars] -> Value #

toEncodingList :: [Dollars] -> Encoding #

omitField :: Dollars -> Bool #

Generic Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Associated Types

type Rep Dollars 
Instance details

Defined in Napkin.Run.BigQuery

type Rep Dollars = D1 ('MetaData "Dollars" "Napkin.Run.BigQuery" "napkin-backend-bigquery-2.0.0-ImbvftYkxiHLT3qYl7zZv" 'True) (C1 ('MetaCons "Dollars" 'PrefixI 'True) (S1 ('MetaSel ('Just "_unDollars") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Fixed E2))))

Methods

from :: Dollars -> Rep Dollars x #

to :: Rep Dollars x -> Dollars #

Num Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

(+) :: Dollars -> Dollars -> Dollars #

(-) :: Dollars -> Dollars -> Dollars #

(*) :: Dollars -> Dollars -> Dollars #

negate :: Dollars -> Dollars #

abs :: Dollars -> Dollars #

signum :: Dollars -> Dollars #

fromInteger :: Integer -> Dollars #

Real Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

toRational :: Dollars -> Rational #

Show Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

showsPrec :: Int -> Dollars -> ShowS #

show :: Dollars -> String #

showList :: [Dollars] -> ShowS #

Eq Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

(==) :: Dollars -> Dollars -> Bool #

(/=) :: Dollars -> Dollars -> Bool #

Ord Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

compare :: Dollars -> Dollars -> Ordering #

(<) :: Dollars -> Dollars -> Bool #

(<=) :: Dollars -> Dollars -> Bool #

(>) :: Dollars -> Dollars -> Bool #

(>=) :: Dollars -> Dollars -> Bool #

max :: Dollars -> Dollars -> Dollars #

min :: Dollars -> Dollars -> Dollars #

type Rep Dollars # 
Instance details

Defined in Napkin.Run.BigQuery

type Rep Dollars = D1 ('MetaData "Dollars" "Napkin.Run.BigQuery" "napkin-backend-bigquery-2.0.0-ImbvftYkxiHLT3qYl7zZv" 'True) (C1 ('MetaCons "Dollars" 'PrefixI 'True) (S1 ('MetaSel ('Just "_unDollars") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Fixed E2))))

calcBytesCost :: Int64 -> Dollars #

data JobInfo #

Constructors

JobInfo 

Fields

Instances

Instances details
ToJSON JobInfo # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

toJSON :: JobInfo -> Value #

toEncoding :: JobInfo -> Encoding #

toJSONList :: [JobInfo] -> Value #

toEncodingList :: [JobInfo] -> Encoding #

omitField :: JobInfo -> Bool #

Generic JobInfo # 
Instance details

Defined in Napkin.Run.BigQuery

Associated Types

type Rep JobInfo 
Instance details

Defined in Napkin.Run.BigQuery

type Rep JobInfo = D1 ('MetaData "JobInfo" "Napkin.Run.BigQuery" "napkin-backend-bigquery-2.0.0-ImbvftYkxiHLT3qYl7zZv" 'False) (C1 ('MetaCons "JobInfo" 'PrefixI 'True) ((S1 ('MetaSel ('Just "jobId") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Text) :*: (S1 ('MetaSel ('Just "startedAt") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 UTCTime) :*: S1 ('MetaSel ('Just "finishedAt") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 UTCTime))) :*: (S1 ('MetaSel ('Just "rowsAffected") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe Int64)) :*: (S1 ('MetaSel ('Just "bytesAffected") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe Int64)) :*: S1 ('MetaSel ('Just "slotTime") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe NominalDiffTime))))))

Methods

from :: JobInfo -> Rep JobInfo x #

to :: Rep JobInfo x -> JobInfo #

Show JobInfo # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

showsPrec :: Int -> JobInfo -> ShowS #

show :: JobInfo -> String #

showList :: [JobInfo] -> ShowS #

Eq JobInfo # 
Instance details

Defined in Napkin.Run.BigQuery

Methods

(==) :: JobInfo -> JobInfo -> Bool #

(/=) :: JobInfo -> JobInfo -> Bool #

type Rep JobInfo # 
Instance details

Defined in Napkin.Run.BigQuery

type Rep JobInfo = D1 ('MetaData "JobInfo" "Napkin.Run.BigQuery" "napkin-backend-bigquery-2.0.0-ImbvftYkxiHLT3qYl7zZv" 'False) (C1 ('MetaCons "JobInfo" 'PrefixI 'True) ((S1 ('MetaSel ('Just "jobId") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Text) :*: (S1 ('MetaSel ('Just "startedAt") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 UTCTime) :*: S1 ('MetaSel ('Just "finishedAt") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 UTCTime))) :*: (S1 ('MetaSel ('Just "rowsAffected") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe Int64)) :*: (S1 ('MetaSel ('Just "bytesAffected") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe Int64)) :*: S1 ('MetaSel ('Just "slotTime") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe NominalDiffTime))))))

calcCost :: MonadNapkin BigQuery m => (UTCTime, Job, UTCTime) -> m Job #

executeQuery :: (MonadNapkin BigQuery m, RenderSql q BigQuery) => BQProjectId -> Labels -> q -> GoogleEnv -> m Job #

bqRunQuery :: (MonadNapkin BigQuery m, RenderSql q BigQuery) => BQProjectId -> Labels -> q -> GoogleEnv -> m [Map Text Value] #

jobQueryStats :: (UTCTime, Job, UTCTime) -> QueryStats BigQuery #

bqCopyTable :: MonadNapkin BigQuery m => BQProjectId -> Labels -> Ref Table -> Ref Table -> TableWriteStrategy -> GoogleEnv -> m () #

bqObtainTableKind :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> GoogleEnv -> m TableKind #

bqAnnotateTable :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> Annotation -> GoogleEnv -> m () #

runInsertJob :: (MonadNapkin BigQuery m, RenderSql a BigQuery) => JobInsertTableName -> BQProjectId -> Labels -> a -> TableMeta -> Ref Table -> GoogleEnv -> m Job #

bqCreateTableAs :: MonadNapkin BigQuery m => BQProjectId -> Labels -> Ref Table -> Query -> TableMeta -> GoogleEnv -> m () #

authorizeView :: MonadNapkin BigQuery m => DataSetLocks -> BQProjectId -> Ref Table -> Set BQDataSetReference -> GoogleEnv -> m () #

createTableDDL :: MonadNapkin BigQuery m => BQProjectId -> Labels -> CreateTableSchema BigQuery -> GoogleEnv -> m () #

jobConfigurationQuery :: Text -> JobConfigurationQuery #

jobConfiguration :: GoogleEnv -> Labels -> JobConfiguration #

jobInsert :: BQProjectId -> JobConfiguration -> BigQueryJobsInsert #

bqGetRelationSchema :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> Relation -> GoogleEnv -> m [BackendSchemaField BigQuery] #

insertAndPollJob :: MonadNapkin BigQuery m => RetryPolicy -> BigQueryJobsInsert -> GoogleEnv -> NominalDiffTime -> BQProjectId -> m (Either (NonEmpty ErrorProto) Job) #

Combines inserting a job and waiting for it to finish. If the job returns *only* server errors, which require retry, it will reenqueue and retry them as per the policy.

pollJob :: GoogleEnv -> NominalDiffTime -> BQProjectId -> JobReference -> IO (Either (NonEmpty ErrorProto) Job) #

Polls a job until it finishes

bqDropTableOrView :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> GoogleEnv -> m () #

bqCheckTableExists :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> GoogleEnv -> m Bool #

streamJobResultsPages :: forall (m :: Type -> Type) i. MonadResource m => GoogleEnv -> Job -> BQProjectId -> ConduitM i GetQueryResultsResponse m () #

streamReq #

Arguments

:: forall req (m :: Type -> Type) i. (AllowRequest req BigQueryScopes, MonadResource m) 
=> GoogleEnv 
-> req 
-> Setter' req (Maybe Text)

Request's page token

-> Getter (Rs req) (Maybe Text)

Response's page token

-> ConduitM i (Rs req) m () 

This probably belongs in gogol. Given an initial request and a way to get the response's next page token and set the requests next page token, repeatedly issue requests until pages run out.

ndtMicros :: NominalDiffTime -> Int #

defDelay :: NominalDiffTime #

executePoll :: MonadNapkin BigQuery m => GoogleEnv -> BigQueryJobsInsert -> BQProjectId -> m (UTCTime, Job, UTCTime) #

tableInfo :: MonadNapkin BigQuery m => BQProjectId -> Ref Table -> m BigQueryTablesGet #

mkBigQueryEnvWithJsonCredentials :: (MonadCatch m, MonadNapkin BigQuery m) => Maybe Text -> CredentialsJson -> DbBackendOptions BigQuery -> BQProjectId -> m (BackendConn BigQuery) #

mkBigQueryEnvWithCredentials :: (MonadCatch m, MonadNapkin BigQuery m) => Maybe Text -> Credentials BigQueryScopes -> DbBackendOptions BigQuery -> BQProjectId -> m (BackendConn BigQuery) #

mkBigQueryEnv' :: (MonadCatch m, MonadIO m) => Natural -> IO GoogleEnv -> BQProjectId -> Labels -> m (BackendConn BigQuery) #

Makes a BigQuery env, but with control over configuring how the Google environment is set up.

data BigQueryURI #

Constructors

BigQueryURI 

Fields

newtype DataSetLock #

Constructors

DataSetLock (MVar ()) 

type DataSetLocks = TVar (Map BQDataSetId DataSetLock) #

getDataSetLock :: MonadIO m => DataSetLocks -> BQDataSetId -> m DataSetLock #

withDataSetLock :: MonadUnliftIO m => DataSetLock -> m a -> m a #

withDataSetLock' :: MonadUnliftIO m => DataSetLocks -> BQDataSetId -> m a -> m a #

parseBigQueryURI :: Text -> Maybe BigQueryURI #

Orphan instances

Backend BigQuery # 
Instance details

Associated Types

data BackendConn BigQuery 
Instance details

Defined in Napkin.Run.BigQuery

data BackendConn BigQuery = BigQueryEnv {}

Methods

injectRunInfo :: RunId -> TaskId -> BackendConn BigQuery -> BackendConn BigQuery #

backendConnectionString :: BackendConn BigQuery -> ConnectionString #

backendExecute :: (MonadNapkin BigQuery m, RenderSql command BigQuery) => BackendConn BigQuery -> command -> m () #

backendQuery :: (MonadNapkin BigQuery m, RenderSql q BigQuery) => BackendConn BigQuery -> q -> m [Map Text Value] #

backendCheckTableExists :: MonadNapkin BigQuery m => BackendConn BigQuery -> Ref Table -> m Bool #

backendGetTableKind :: MonadNapkin BigQuery m => BackendConn BigQuery -> Ref Table -> m TableKind #

backendGetRelationSchema :: MonadNapkin BigQuery m => BackendConn BigQuery -> Ref Table -> Relation -> m [BackendSchemaField BigQuery] #

backendGetAnnotations :: MonadNapkin BigQuery m => BackendConn BigQuery -> Ref Table -> m TableAnnotations #

backendAnnotate :: MonadNapkin BigQuery m => BackendConn BigQuery -> Ref Table -> Annotation -> m () #

backendCreateAs :: (MonadNapkin BigQuery m, RenderSql command BigQuery, command ~ CreateRelation BigQuery) => BackendConn BigQuery -> command -> m () #

backendCreate :: (MonadNapkin BigQuery m, RenderSql command BigQuery, command ~ CreateTableSchema BigQuery) => BackendConn BigQuery -> command -> m () #

backendCopyTable :: (Default (BackendMeta 'KindTable BigQuery), MonadNapkin BigQuery m, RenderSql (CreateRelation BigQuery) BigQuery, RenderSql DeleteRows BigQuery, RenderSql InsertInto BigQuery) => BackendConn BigQuery -> Ref Table -> Ref Table -> TableWriteStrategy -> m () #

backendDrop :: MonadNapkin BigQuery m => BackendConn BigQuery -> DropRelation -> m () #

backendInsertInto :: (MonadNapkin BigQuery m, RenderSql command BigQuery, command ~ InsertInto) => BackendConn BigQuery -> command -> m () #

backendRename :: MonadNapkin BigQuery m => BackendConn BigQuery -> RenameRelation -> m () #

backendListTables :: MonadNapkin BigQuery m => BackendConn BigQuery -> ListTables -> m (Set ListedTable) #

backendNormalize :: MonadNapkin BigQuery m => BackendConn BigQuery -> NormalizeTableNames -> m (Map (Ref Table) NormalizedTable) #

backendAddColumn :: (MonadNapkin BigQuery m, RenderSql command BigQuery, command ~ AddColumnToTable) => BackendConn BigQuery -> command -> m () #

backendDropColumn :: (MonadNapkin BigQuery m, RenderSql command BigQuery, command ~ DropColumnFromTable) => BackendConn BigQuery -> command -> m () #

RunBackendConn BigQuery # 
Instance details

Methods

runBackendConn :: LogEnv -> DbBackendOptions BigQuery -> ConnectionString -> Maybe CredentialsPath -> Maybe AuthPort -> (BackendConn BigQuery -> IO a) -> IO a #

HasBackendSchemaField BigQuery # 
Instance details

Associated Types

data BackendSchemaField BigQuery 
Instance details

Defined in Napkin.Run.BigQuery

data BackendSchemaField BigQuery = BigQuerySchemaField TableFieldSchema

Methods

fieldName :: BackendSchemaField BigQuery -> Text #

fieldType :: BackendSchemaField BigQuery -> Type #

HasBackendQueryStats BigQuery # 
Instance details

Associated Types

data BackendQueryStats BigQuery 
Instance details

Defined in Napkin.Run.BigQuery

data BackendQueryStats BigQuery = BackendQueryStats {}

Methods

prettyPrint :: BackendQueryStats BigQuery -> Maybe Text #

Generic (BackendConn BigQuery) # 
Instance details

Associated Types

type Rep (BackendConn BigQuery) 
Instance details

Defined in Napkin.Run.BigQuery

type Rep (BackendConn BigQuery) = D1 ('MetaData "BackendConn" "Napkin.Run.BigQuery" "napkin-backend-bigquery-2.0.0-ImbvftYkxiHLT3qYl7zZv" 'False) (C1 ('MetaCons "BigQueryEnv" 'PrefixI 'True) ((S1 ('MetaSel ('Just "envPool") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Pool GoogleEnv)) :*: S1 ('MetaSel ('Just "defaultProject") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 BQProjectId)) :*: (S1 ('MetaSel ('Just "labels") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Labels) :*: S1 ('MetaSel ('Just "dataSetLocks") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 DataSetLocks))))

Methods

from :: BackendConn BigQuery -> Rep (BackendConn BigQuery) x #

to :: Rep (BackendConn BigQuery) x -> BackendConn BigQuery #

ToJSON (BackendQueryStats BigQuery) # 
Instance details

Methods

toJSON :: BackendQueryStats BigQuery -> Value #

toEncoding :: BackendQueryStats BigQuery -> Encoding #

toJSONList :: [BackendQueryStats BigQuery] -> Value #

toEncodingList :: [BackendQueryStats BigQuery] -> Encoding #

omitField :: BackendQueryStats BigQuery -> Bool #

Monoid (BackendQueryStats BigQuery) # 
Instance details

Methods

mempty :: BackendQueryStats BigQuery #

mappend :: BackendQueryStats BigQuery -> BackendQueryStats BigQuery -> BackendQueryStats BigQuery #

mconcat :: [BackendQueryStats BigQuery] -> BackendQueryStats BigQuery #

Semigroup (BackendQueryStats BigQuery) # 
Instance details

Methods

(<>) :: BackendQueryStats BigQuery -> BackendQueryStats BigQuery -> BackendQueryStats BigQuery #

sconcat :: NonEmpty (BackendQueryStats BigQuery) -> BackendQueryStats BigQuery #

stimes :: Integral b => b -> BackendQueryStats BigQuery -> BackendQueryStats BigQuery #

Show (BackendSchemaField BigQuery) # 
Instance details

Methods

showsPrec :: Int -> BackendSchemaField BigQuery -> ShowS #

show :: BackendSchemaField BigQuery -> String #

showList :: [BackendSchemaField BigQuery] -> ShowS #

Show (BackendQueryStats BigQuery) # 
Instance details

Methods

showsPrec :: Int -> BackendQueryStats BigQuery -> ShowS #

show :: BackendQueryStats BigQuery -> String #

showList :: [BackendQueryStats BigQuery] -> ShowS #

Eq (BackendSchemaField BigQuery) # 
Instance details

Methods

(==) :: BackendSchemaField BigQuery -> BackendSchemaField BigQuery -> Bool #

(/=) :: BackendSchemaField BigQuery -> BackendSchemaField BigQuery -> Bool #

Eq (BackendQueryStats BigQuery) # 
Instance details

Methods

(==) :: BackendQueryStats BigQuery -> BackendQueryStats BigQuery -> Bool #

(/=) :: BackendQueryStats BigQuery -> BackendQueryStats BigQuery -> Bool #