Skip to content

Instantly share code, notes, and snippets.

@pondzix
Last active July 23, 2024 07:55
Show Gist options
  • Save pondzix/bf5355dc82096593a72dc80bc41a87b7 to your computer and use it in GitHub Desktop.
Save pondzix/bf5355dc82096593a72dc80bc41a87b7 to your computer and use it in GitHub Desktop.
BigQuery - fields with names starting with a number have `null` value after write
//> using dep com.google.cloud:google-cloud-bigquerystorage:3.7.0
//> using dep com.google.cloud:google-cloud-bigquery:2.41.0
import com.google.cloud.bigquery.TableId
import com.google.cloud.bigquery.storage.v1.{BigQueryWriteClient, JsonStreamWriter}
import org.json.JSONArray
import scala.jdk.CollectionConverters._
/**
CREATE TABLE
test_dataset.test_table (
parent1 STRUCT< `1nested` STRING >,
parent2 STRUCT< `1nested` STRING >
)
*/
val input = Map(
"parent1" -> Map("1nested" -> "value").asJava, // "value" in BQ
"parent2" -> Map("1nested" -> "value").asJava, // null in BQ
)
write(input)
def write(input: Map[String, AnyRef]): Unit = {
val client = BigQueryWriteClient.create()
val streamId = TableId.of("...project..", "...dataset..", "..table..").getIAMResourceName + "/streams/_default"
val writer = JsonStreamWriter
.newBuilder(streamId, client)
.build
writer.append(new JSONArray(List(input.asJava).asJava)).get
writer.close()
client.close()
()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment