☁️ Azure Data Factory Custom Dataset Terraform Module

module "dataset" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "ds_landing"
  data_factory_id = var.data_factory_id
  type            = "AzureBlob"
  linked_service  = { name = "ls-blob" }

  type_properties_json = jsonencode({
    folderPath = "raw"
    container  = "landing"
  })
}

# WRONG -- the provider adds the envelope itself, so this nests twice:
#
#   type_properties_json = jsonencode({
#     typeProperties = { folderPath = "raw" }
#   })

check "no_double_wrap" {
  assert {
    condition     = !module.dataset.type_properties_looks_double_wrapped
    error_message = "type_properties_json already contains a typeProperties key. The provider wraps this value itself, so the payload will be nested twice and Azure will not interpret it as intended."
  }
}

check "blob_properties_present" {
  assert {
    condition = alltrue([
      for k in ["folderPath", "container"] : contains(module.dataset.type_property_keys, k)
    ])
    error_message = "The AzureBlob dataset is missing folderPath or container. Nothing in the provider or the module checks the JSON shape, so assert on the keys here."
  }
}

module "awkward_name" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "my/dataset:v1" # ACCEPTED -- by the provider and by this module
  data_factory_id = var.data_factory_id
  type            = "AzureBlob"
  linked_service  = { name = "ls-blob" }
  type_properties_json = jsonencode({ folderPath = "raw" })
}

output "the_defect" {
  value = module.awkward_name.the_provider_name_validator_is_effectively_inert
}

output "name_check" {
  value = module.dataset.name_contains_characters_azure_may_reject
}

module "typed_dataset" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "ds_customers"
  data_factory_id = var.data_factory_id
  type            = "DelimitedText"
  linked_service  = { name = "ls-blob" }

  type_properties_json = jsonencode({
    location = { type = "AzureBlobStorageLocation", container = "curated", fileName = "customers.csv" }
    columnDelimiter  = ","
    firstRowAsHeader = true
  })

  schema_json = jsonencode([
    { name = "id", type = "String" },
    { name = "created_at", type = "DateTime" },
  ])
}

module "parameterised" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "ds_by_date"
  data_factory_id = var.data_factory_id
  type            = "AzureBlob"

  linked_service = {
    name       = "ls-blob"
    parameters = { environment = "prod" }
  }

  parameters = {
    partitionDate = "2026-01-01"
  }

  type_properties_json = jsonencode({
    folderPath = "@dataset().partitionDate"
    container  = "landing"
  })
}

module "with_extras" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "ds_landing"
  data_factory_id = var.data_factory_id
  type            = "AzureBlob"
  linked_service  = { name = "ls-blob" }
  type_properties_json = jsonencode({ folderPath = "raw" })

  additional_properties = {
    someUnmodelledField = "value"
  }
}

check "escape_hatch_is_deliberate" {
  assert {
    condition     = !module.with_extras.uses_additional_properties
    error_message = "This dataset uses additional_properties, which writes arbitrary top-level keys into the definition. Confirm it is intended -- a key colliding with a managed field is a conflict nothing reports."
  }
}

output "no_diff_on_reformat" {
  value = module.dataset.json_key_ordering_produces_no_diff
}

locals {
  containers = {
    landing = "raw"
    curated = "curated"
  }
}

module "datasets" {
  source   = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"
  for_each = local.containers

  name            = "ds_${each.key}"
  data_factory_id = var.data_factory_id
  type            = "AzureBlob"
  linked_service  = { name = "ls-blob" }

  type_properties_json = jsonencode({
    folderPath = each.value
    container  = each.key
  })
}

output "property_counts" {
  value = { for k, m in module.datasets : k => m.type_property_count }
}

output "unverifiable" {
  value = {
    linked_service = module.dataset.the_linked_service_is_not_verified_to_exist
    json_shape     = module.dataset.the_json_shape_is_not_validated_by_anything
    type_pairing   = module.dataset.the_type_and_the_json_are_coupled_and_unchecked
    consumers      = module.dataset.the_module_cannot_see_which_pipelines_use_this
  }
}

output "change_surface" {
  value = {
    force_new = module.dataset.force_new_fields                      # name, factory, type
    in_place  = module.dataset.fields_that_can_change_after_creation # everything else
  }
}

import {
  to = module.dataset.azurerm_data_factory_custom_dataset.this
  id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-data-eastus/providers/Microsoft.DataFactory/factories/adf-corp/datasets/ds_landing"
}

provider "azurerm" {
  features {}
}

# 1. The factory -- the parent, and the only taggable resource here.
module "data_factory" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory.git?ref=v1.0.0"

  name                = "adf-corp"
  resource_group_name = "rg-data-eastus"
  location            = "eastus"

  tags = { owner = "data-platform" }
}

# 2. A credential the linked service will authenticate with. No secret passes
#    through Terraform -- it names a Key Vault secret.
module "credential" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-credential-user-managed-identity.git?ref=v1.0.0"

  name            = "cred-managed"
  data_factory_id = module.data_factory.id
  identity_id     = var.user_assigned_identity_id

  depends_on = [module.data_factory]
}

# 3. THIS MODULE -- a description of where data lives. It reads nothing.
module "dataset" {
  source = "git::https://github.com/microsoftexpert/terraform-azurerm-data-factory-custom-dataset.git?ref=v1.0.0"

  name            = "ds_landing"
  data_factory_id = module.data_factory.id
  type            = "AzureBlob"

  # The linked service is named by STRING and is not created here -- it is
  # configured in the factory and referenced by name.
  linked_service = { name = "ls-blob" }

  type_properties_json = jsonencode({
    folderPath = "raw"
    container  = "landing"
  })

  description = "Landing-zone blobs written by the ingest pipeline."
  folder      = "raw"
  annotations = ["owner:data-platform"]
}

check "json_has_the_expected_keys" {
  assert {
    condition = alltrue([
      for k in ["folderPath", "container"] : contains(module.dataset.type_property_keys, k)
    ])
    error_message = "The AzureBlob dataset is missing folderPath or container."
  }
}

check "no_double_wrap" {
  assert {
    condition     = !module.dataset.type_properties_looks_double_wrapped
    error_message = "type_properties_json already contains a typeProperties key; the provider adds that envelope itself."
  }
}

output "dataset_status" {
  description = "What is described, and what Terraform deliberately cannot confirm."
  value = {
    dataset         = module.dataset.id
    used_by_name    = module.dataset.name
    properties      = module.dataset.type_property_keys
    linked_service  = module.dataset.linked_service_name
    credential      = module.credential.name
    reads_anything  = false # A dataset is a description; pipelines do the work.
  }
}

variable "name" {
  type = string
  # Non-empty, and refuses a name made ENTIRELY of - . + ? / < > * % & : \
  # -- which is the only thing the provider actually rejects. A name merely
  # CONTAINING them is accepted, and reported.
}

variable "data_factory_id" {
  type = string
  # /subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.DataFactory
  #   /factories/<factory>
}

variable "type" {
  type = string
  # Non-empty only. NO enum -- the dataset type set is Microsoft's and grows.
  # Decides what type_properties_json must contain; the pairing is unchecked.
}

variable "linked_service" {
  type = object({
    name       = string                 # a NAME in this factory, not an ID
    parameters = optional(map(string))
  })
  # A single object: the provider permits one block, a limit absent from its
  # published schema.
}

variable "type_properties_json" {
  type = string
  # Checked for PARSEABILITY and that it decodes to an OBJECT. Nothing else.
  # SUPPLY THE INNER OBJECT -- the provider adds a typeProperties envelope.
}

variable "schema_json" {
  type    = string
  default = null
  # Parseability only. Sent as supplied, with no envelope.
}

variable "parameters" {
  type    = map(string)
  default = {}
}

variable "additional_properties" {
  type    = map(string)
  default = {}
  # An escape hatch: arbitrary top-level keys, able to collide with managed
  # fields without either Terraform or the provider reporting it.
}

variable "annotations" {
  type    = list(string)
  default = []
  # Data Factory annotations -- a LIST, unrelated to Azure resource tags.
}

variable "description" {
  type    = string
  default = null
}

variable "folder" {
  type    = string
  default = null
  # Omit for the root level. Purely an authoring convenience.
}

variable "timeouts" {
  type = object({
    create = optional(string)
    read   = optional(string)
    update = optional(string)
    delete = optional(string)
  })
  default = null
  # Provider defaults: create 30m, update 30m, read 5m, delete 30m.
}