Data Management Fundamentals 💾

Master the essentials of managing your data in Dataloop - from uploading files to organizing your datasets efficiently.

Project Setup ⚙️

import dtlpy as dl

# Interactive login — opens a browser window
if dl.token_expired():
    dl.login()

Project and Dataset Setup

# Set your project and dataset names
project_name = "onboarding-project"
dataset_name = "onboarding-dataset"
try:
    # Try to get existing project
    project = dl.projects.get(project_name=project_name)
    print(f"Project '{project_name}' already exists")
except dl.exceptions.NotFound:
    project = dl.projects.create(project_name=project_name)
    # Create project if it doesn't exist
    print(f"Created project '{project_name}'")

try:
    # Try to get existing dataset
    dataset = project.datasets.get(dataset_name=dataset_name)
    print(f"Dataset '{dataset_name}' already exists")

except dl.exceptions.NotFound:
    # Create dataset if it doesn't exist
    dataset = project.datasets.create(dataset_name=dataset_name)
    print(f"Created dataset '{dataset_name}'")

Dataset Cleanup

# Empty dataset - delete all dataset items
for item in dataset.items.list().all():
    item.delete()

Uploading Items 📤

1. Single Item Upload

# Upload a single file
item = dataset.items.upload(
    # Set the local JPG file path — use r prefix if the path contains special characters, e.g., r'c:\users\one drive\dog.jpg'
    local_path='path/to/your/file.jpg',
    remote_path='remote/path/',  # Optional
    item_metadata={
        'photographer': 'John Doe',
        'location': 'New York'
    }
)

# Store item ID for later use in notebook
single_item_id = item.id

item.metadata['user'] = {
    'name': 'Lucky Luke',
    'location': 'Belgium'
}

item.update()

# Explore item
item.print()

print(item.metadata)

# Explore item metadata in Dataloop platform
item.open_in_web()

2. Batch Upload

# Upload entire directory
dataset.items.upload(
    # Set the local folder path
    local_path='/path/to/folder',
    remote_path='/batch-upload',
    local_annotations_path='/path/to/annotations'  # Optional
)

# Upload multiple specific files
local_image_1 = 'path/to/your/file_1.jpg'
local_image_2 = 'path/to/your/file_2.jpg'

items = dataset.items.upload(
    local_path=[local_image_1, local_image_2],
    remote_path='/batch-upload'
)

# List items in the dataset
dataset.items.list().print()

# Explore dataset in Dataloop platform
dataset.open_in_web()

File Organization Strategies 📂

1. Directory Structure

# Get item
item = dataset.items.get(item_id=single_item_id)

# Print item details
item.print()

# Move items
item.move(
    new_path='/new/path/item.jpg'
)

# Print item details after the move
item.print()

# List directory contents after setting filter value
dataset.items.list(filters=dl.Filters(field='dir', values='/batch-upload')).print()

2. Item Management

# Set your filter directory value
dataset.items.list().print()

# ⚠️ Skip this cell if you plan to continue to the next chapters — it deletes items needed later
# Delete items
dataset.items.delete(filters=dl.Filters(field='dir', values='/batch-upload'))
dataset.items.list().print()

3. Metadata Organization

item = dataset.items.get(item_id=single_item_id)
item.print()

# Add metadata to item
item.metadata['user'] = {
    'status': 'reviewed',
    'quality': 'high',
    'tags': ['validated', 'ready']
}
item = item.update()

print(item.metadata['user'])

dataset.items.list().print()

# Set the directory filter value (use the 'dir' column from the items list)
filter_items_dir = '/dataset/folder'
filters = dl.Filters(field='dir', values=filter_items_dir)
# Batch metadata update
dataset.items.update(
    filters=filters,
    update_values={'user.status': 'reviewed_in_batch'}
)

Batch Operations ⚡

1. Bulk Upload and Download

# Bulk upload

# Set local folder path
local_folder_path = '/path/to/folder'

dataset.items.upload(
    local_path=local_folder_path,
    remote_path='/dataset/folder'
)


# Download items - set download local path
dataset.items.download(
    local_path='/path/to/folder_download',
    filters=dl.Filters(field='dir', values='/batch-upload')
)

# Print dataset items after upload
dataset.items.list().print()

2. Batch Processing

# Process multiple items after setting the filter value
filters = dl.Filters(field='dir', values='/batch-upload')
pages = dataset.items.list(filters=filters)

def process_item(item):
    """Process a single item"""
    print(f'Processing: {item.name}')
    # Add your processing logic here
    # e.g., download, transform, analyze
    return item

for page in pages:
    for item in page:
        # Process each item
        process_item(item)

# Print list before delete
dataset.items.list().print()

# ⚠️ Skip this delete if you plan to continue to the next chapters — it deletes items needed later
# Batch delete
dataset.items.delete(filters=filters)

# Print list after delete
dataset.items.list().print()

3. Concurrent Operations

import concurrent.futures

# Get all items for concurrent processing
items = list(dataset.items.list().all())

with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
    futures = [executor.submit(process_item, item) for item in items]
    concurrent.futures.wait(futures)

Data Validation 🔍

1. Item Validation

# Check item existence
try:
    item = dataset.items.get(filepath='/path/to/item.jpg')
except dl.exceptions.NotFound:
    print("Item not found!")

# Validate item metadata
def validate_item(item):
    required_fields = ['status', 'quality']
    metadata = item.metadata.get('user', {})
    return all(field in metadata for field in required_fields)

2. Batch Validation

# Validate multiple items
def validate_items(dataset):
    invalid_items = []
    for item in dataset.items.list().all():
        if not validate_item(item):
            invalid_items.append(item.id)
    return invalid_items

3. Data Integrity Checks

from PIL import Image

# Check for corrupted images
def check_image_integrity(item):
    try:
        buffer = item.download(save_locally=False)
        Image.open(buffer)
        return True
    except Exception as e:
        print(f"Corrupted image {item.name}: {e}")
        return False

Pro Tips 💡

Efficient Data Organization

# Create folders using make_dir
folders = ['train', 'val', 'test']
for folder in folders:
    dataset.items.make_dir(directory=f'/{folder}')

Ready to start annotating your data? Let's move on to the annotation chapter! 🎯