Skip to content

Commit 0ff71e6

Browse files
authored
feat(bigquery): adds sql query for VIEW and MATERIALIZED VIEW in asset for BQ (#488)
* feat: group comamnds * feat(bigquery): adds sql query for VIEW and MATERIALIZED VIEW in asset for BQ
1 parent d564772 commit 0ff71e6

File tree

1 file changed

+30
-27
lines changed

1 file changed

+30
-27
lines changed

plugins/extractors/bigquery/bigquery.go

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -415,44 +415,47 @@ func (e *Extractor) buildAsset(ctx context.Context, t *bigquery.Table, md *bigqu
415415
}
416416
}
417417

418-
table, err := anypb.New(&v1beta2.Table{
419-
Columns: e.buildColumns(ctx, md.Schema, md),
420-
PreviewFields: previewFields,
421-
PreviewRows: previewRows,
422-
Profile: tableProfile,
423-
Attributes: utils.TryParseMapToProto(map[string]interface{}{
424-
"full_qualified_name": tableFQN,
425-
"dataset": t.DatasetID,
426-
"project": t.ProjectID,
427-
"type": string(md.Type),
428-
"partition_data": partitionData,
429-
"clustering_fields": clusteringFields,
430-
}),
431-
CreateTime: timestamppb.New(md.CreationTime),
432-
UpdateTime: timestamppb.New(md.LastModifiedTime),
433-
})
434-
if err != nil {
435-
e.logger.Warn("error creating Any struct", "error", err)
436-
}
437-
438418
asset := &v1beta2.Asset{
439419
Urn: tableURN,
440420
Name: t.TableID,
441421
Type: "table",
442422
Description: md.Description,
443423
Service: "bigquery",
444-
Data: table,
445-
Labels: md.Labels,
424+
// Data: table,
425+
Labels: md.Labels,
426+
}
427+
attributesData := map[string]interface{}{
428+
"full_qualified_name": tableFQN,
429+
"dataset": t.DatasetID,
430+
"project": t.ProjectID,
431+
"type": string(md.Type),
432+
"partition_data": partitionData,
433+
"clustering_fields": clusteringFields,
446434
}
447435

448-
if e.config.BuildViewLineage && (md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView) {
436+
if md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView {
449437
query := getViewQuery(md)
450-
upstreamResources := getUpstreamResources(query)
451-
asset.Lineage = &v1beta2.Lineage{
452-
Upstreams: upstreamResources,
438+
attributesData["sql"] = query
439+
if e.config.BuildViewLineage {
440+
upstreamResources := getUpstreamResources(query)
441+
asset.Lineage = &v1beta2.Lineage{
442+
Upstreams: upstreamResources,
443+
}
453444
}
454445
}
455-
446+
table, err := anypb.New(&v1beta2.Table{
447+
Columns: e.buildColumns(ctx, md.Schema, md),
448+
PreviewFields: previewFields,
449+
PreviewRows: previewRows,
450+
Profile: tableProfile,
451+
Attributes: utils.TryParseMapToProto(attributesData),
452+
CreateTime: timestamppb.New(md.CreationTime),
453+
UpdateTime: timestamppb.New(md.LastModifiedTime),
454+
})
455+
if err != nil {
456+
e.logger.Warn("error creating Any struct", "error", err)
457+
}
458+
asset.Data = table
456459
return asset, nil
457460
}
458461

0 commit comments

Comments
 (0)