Getting Started
/
Metadata and Filtering
Last updated

Metadata and Filtering: Organizing Your AI Data 🔍

Master the art of organizing and finding your data using Dataloop's powerful metadata and filtering capabilities.

Project Setup ⚙️

Dataloop Login 🔐

import dtlpy as dl

# Interactive login — opens a browser window
if dl.token_expired():
    dl.login()

Project and Dataset Setup

# Set your project and dataset names
project_name = "onboarding-project"
dataset_name = "onboarding-dataset"

try:
    # Try to get existing project
    project = dl.projects.get(project_name=project_name)
    print(f"Project '{project_name}' already exists")
except dl.exceptions.NotFound:
    project = dl.projects.create(project_name=project_name)
    # Create project if it doesn't exist
    print(f"Created project '{project_name}'")

try:
    # Try to get existing dataset
    dataset = project.datasets.get(dataset_name=dataset_name)
    print(f"Dataset '{dataset_name}' already exists")

except dl.exceptions.NotFound:
    # Create dataset if it doesn't exist
    dataset = project.datasets.create(dataset_name=dataset_name)
    print(f"Created dataset '{dataset_name}'")

Dataset Items Setup

# Ensure the dataset has items — if empty, upload sample files
if dataset.items.list().items_count == 0:
    print("Dataset is empty. Please upload some items before proceeding.")
    print("You can upload items by running: dataset.items.upload(local_path='path/to/your/files')")
else:
    print(f"Dataset has {dataset.items.list().items_count} items")

Working with Metadata 📝

1. Adding Metadata

# Print dataset items details
dataset.items.list().print()
# Print dataset items metadata
for item in dataset.items.list().all():
    print(item.metadata)
# Get first item id
item = dataset.items.list().items[0]
item_id = item.id
# Add metadata to the first item
item = dataset.items.get(item_id=item_id)
item.metadata['user'] = {
    'photographer': 'John Doe',
    'location': 'New York',
    'camera': {
        'model': 'Canon EOS R5',
        'settings': {
            'iso': 120,
            'aperture': 'f/2.8',
            'shutter_speed': '1/1000'
        }
    },
    'tags': ['outdoor', 'daylight']
}
item = item.update()

print(item.metadata['user'])
# Set the item local path 
local_image_path = '/path/to/image.jpg'

# Add metadata during upload
item = dataset.items.upload(
    local_path=local_image_path,
    item_metadata={
        'user': {
            'project_id': 'PRJ-123',
            'batch': 'B-001'
        }
    }
)

for item in dataset.items.list().all():
    print(item.metadata)

2. Updating Metadata

dataset.items.list().print()
# Update specific fields
item.metadata['user']['status'] = 'reviewed'
item.metadata['user']['last_modified'] = '2024-03-20'
item = item.update()

# Set the filter dir value to match a folder in your dataset (run dataset.items.list().print() to see available dirs)
filters = dl.Filters(field='dir', values='/your-folder/')
dataset.items.update(
    filters=filters,
    update_values={
        'user.status': 'processed',
        'user.batch': 'B-001'
    }
)
for item in dataset.items.list().all():
    print(item.metadata)

Advanced Filtering 🎯

1. Basic Filters

dataset.items.list().print()
# Create filters
filters = dl.Filters()

# Filter by filename
filters.add(field='filename', values='*.jpg')


# Filter by directory — set this to a folder in your dataset
filters.add(field='dir', values='/your-folder')

# Filter by created date — adjust this date to match your data
filters.add(field='createdAt', values='2026-05-23', operator=dl.FiltersOperations.GREATER_THAN)

dataset.items.list(filters=filters).print()

2. Metadata Filters

for item in dataset.items.list().all():
    print(item.metadata)
# Filter by metadata fields
filters = dl.Filters()

# Exact match
filters.add(field='metadata.user.location', values='New York')

# Larger than, smaller than
filters.add(field='metadata.user.camera.settings.iso',
           values=100,
           operator=dl.FiltersOperations.GREATER_THAN_OR_EQUAL)

dataset.items.list(filters=filters).print()
for item in dataset.items.list(filters=filters).all():
    print(item.metadata)

3. Complex Queries

for item in dataset.items.list().all():
    print(item.metadata)
# Combining multiple filters
filters = dl.Filters(resource=dl.FiltersResource.ITEM)

# AND operation (default)
filters.add(field='metadata.user.status', values='reviewed')
filters.add(field='metadata.user.batch', values='B-001')

# NOT EQUAL operation
filters.add(field='metadata.user.status', values='rejected', operator=dl.FiltersOperations.NOT_EQUAL)

print(filters.prepare())

dataset.items.list(filters=filters).print()
for item in dataset.items.list(filters=filters).all():
    print(item.metadata)
dataset.open_in_web()

4. Pagination and Sorting

# Get items with pagination
filters = dl.Filters()
pages = dataset.items.list(
    filters=filters,
    page_offset=0,
    page_size=50
)

# Sort results
filters.sort_by(field='metadata.user.quality', value=dl.FiltersOrderByDirection.ASCENDING)

Filter Error Troubleshooting

# Filter by metadata fields
filters = dl.Filters()
filters.add(field='metadata.user.camera',
           values=True,
           operator=dl.FiltersOperations.EXISTS)

dataset.items.list(filters=filters).print()

Filter Error Message:

"Cannot query on key 'your-key' - no items contain the specified key, or the key is unsearchable"

Reason: This error occurs when the filter key is either invalid or not queryable.

Troubleshooting: The dataset schema defines all filterable keys, you can verify your filter by checking the schema. Check the dataset schema endpoint to see if your key is listed in the schema keys or unsearchablePaths.

Dataset Schema Endpoint:

https://gate.dataloop.ai/api/v1/datasets/<dataset_id>/schema

Check for your key under:

  • schema keys – filtered keys
  • unsearchablePaths – unsearchable paths

Ready to explore task management? Let's move on to the next chapter! 🚀

Previous page
Next page