Moderation

Animus provides content moderation capabilities that help you identify and filter potentially harmful content. Unlike some other APIs, our moderation is integrated directly into our chat completions endpoint using the compliance parameter.

Using Content Moderation

To enable content moderation in your API requests, simply set the compliance parameter to true in your chat completion requests. When enabled, the API will analyze the content and return details about any detected violations.

// Example using fetch API
const response = await fetch('https://api.animusai.co/v2/chat/completions', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': `Bearer ${process.env.ANIMUS_API_KEY}`
  },
  body: JSON.stringify({
    model: "vivian-llama3.1-70b-1.0-fp8",
    messages: [
      { role: "user", content: "Text to check for compliance" }
    ],
    compliance: true  // Enable content moderation
  })
});

const data = await response.json();

// Check if there are any compliance violations
if (data.compliance_violations && data.compliance_violations.length > 0) {
  console.log("Content violations detected:", data.compliance_violations);
} else {
  console.log("No content violations detected");
}

import requests

# API endpoint
url = "https://api.animusai.co/v2/chat/completions"

# Request headers
headers = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {API_KEY}"  # Replace with your API key
}

# Request payload
payload = {
    "model": "vivian-llama3.1-70b-1.0-fp8",
    "messages": [
        {
            "role": "user",
            "content": "Text to check for compliance"
        }
    ],
    "compliance": True  # Enable compliance checking
}

# Make the request
response = requests.post(url, headers=headers, json=payload)
data = response.json()

# Check for compliance violations
if "compliance_violations" in data and data["compliance_violations"]:
    print("Content violations detected:", data["compliance_violations"])
else:
    print("No content violations detected")

curl https://api.animusai.co/v2/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $ANIMUS_API_KEY" \
  -d '{
    "model": "vivian-llama3.1-70b-1.0-fp8",
    "messages": [
      {"role": "user", "content": "Text to check for compliance"}
    ],
    "compliance": true
  }'

Response Format

When compliance checking is enabled, the API response will include a compliance_violations field that contains an array of any detected content violations. Here’s an example response with detected violations:

{
  "id": "chat-abcd1234",
  "object": "chat.completion",
  "created": 1677652288,
  "model": "vivian-llama3.1-70b-1.0-fp8",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "Response content here..."
      }
    }
  ],
  "usage": {
    "prompt_tokens": 40,
    "completion_tokens": 60,
    "total_tokens": 100
  },
  "compliance_violations": ["drug_use"]
}

Content Violation Categories

Our compliance system can detect and flag the following categories of potentially harmful content:

Category	Description
pedophilia	Content related to sexual content involving minors
beastiality	Content involving sexual acts with animals
murder	Content that promotes or glorifies murder
rape	Content related to sexual assault
incest	Content involving sexual relations between family members
gore	Explicit and graphic violent content
prostitution	Content promoting or soliciting prostitution
drug_use	Content promoting or describing drug use

Advanced Implementation

Handling Different Violation Types

async function handleContentModeration(userContent) {
  const response = await fetch('https://api.animusai.co/v2/chat/completions', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer ${process.env.ANIMUS_API_KEY}`
    },
    body: JSON.stringify({
      model: "vivian-llama3.1-70b-1.0-fp8",
      messages: [
        { role: "user", content: userContent }
      ],
      compliance: true
    })
  });
  
  const data = await response.json();
  
  if (data.compliance_violations && data.compliance_violations.length > 0) {
    // Handle different types of violations
    const violations = data.compliance_violations;
    
    if (violations.includes('drug_use')) {
      console.log("Drug-related content detected");
      // Implement specific handling for drug content
    }
    
    if (violations.includes('gore') || violations.includes('murder')) {
      console.log("Violent content detected");
      // Implement specific handling for violent content
    }
    
    // Log for review
    logViolation(userContent, violations);
    
    return {
      allowed: false,
      violations: violations,
      message: "Content violates our community guidelines"
    };
  }
  
  return {
    allowed: true,
    content: data.choices[0].message.content
  };
}

function logViolation(content, violations) {
  // Log violation for review and analysis
  console.log(`Violation logged: ${violations.join(', ')} - Content: ${content.substring(0, 100)}...`);
}

import requests
import logging

def handle_content_moderation(user_content):
    url = "https://api.animusai.co/v2/chat/completions"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {API_KEY}"
    }
    
    payload = {
        "model": "vivian-llama3.1-70b-1.0-fp8",
        "messages": [
            {"role": "user", "content": user_content}
        ],
        "compliance": True
    }
    
    response = requests.post(url, headers=headers, json=payload)
    data = response.json()
    
    if "compliance_violations" in data and data["compliance_violations"]:
        violations = data["compliance_violations"]
        
        # Handle different types of violations
        if "drug_use" in violations:
            print("Drug-related content detected")
            # Implement specific handling for drug content
        
        if "gore" in violations or "murder" in violations:
            print("Violent content detected")
            # Implement specific handling for violent content
        
        # Log for review
        log_violation(user_content, violations)
        
        return {
            "allowed": False,
            "violations": violations,
            "message": "Content violates our community guidelines"
        }
    
    return {
        "allowed": True,
        "content": data["choices"][0]["message"]["content"]
    }

def log_violation(content, violations):
    """Log violation for review and analysis"""
    logging.warning(f"Violation logged: {', '.join(violations)} - Content: {content[:100]}...")

package main

import (
    "bytes"
    "encoding/json"
    "fmt"
    "net/http"
    "strings"
)

type ModerationRequest struct {
    Model      string    `json:"model"`
    Messages   []Message `json:"messages"`
    Compliance bool      `json:"compliance"`
}

type Message struct {
    Role    string `json:"role"`
    Content string `json:"content"`
}

type ModerationResponse struct {
    Choices              []Choice  `json:"choices"`
    ComplianceViolations []string  `json:"compliance_violations"`
}

type Choice struct {
    Message Message `json:"message"`
}

type ModerationResult struct {
    Allowed    bool     `json:"allowed"`
    Violations []string `json:"violations,omitempty"`
    Content    string   `json:"content,omitempty"`
    Message    string   `json:"message,omitempty"`
}

func handleContentModeration(userContent string) (*ModerationResult, error) {
    reqBody := ModerationRequest{
        Model: "vivian-llama3.1-70b-1.0-fp8",
        Messages: []Message{
            {Role: "user", Content: userContent},
        },
        Compliance: true,
    }
    
    jsonData, err := json.Marshal(reqBody)
    if err != nil {
        return nil, err
    }
    
    req, err := http.NewRequest("POST", "https://api.animusai.co/v2/chat/completions", bytes.NewBuffer(jsonData))
    if err != nil {
        return nil, err
    }
    
    req.Header.Set("Content-Type", "application/json")
    req.Header.Set("Authorization", "Bearer "+apiKey)
    
    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()
    
    var modResp ModerationResponse
    if err := json.NewDecoder(resp.Body).Decode(&modResp); err != nil {
        return nil, err
    }
    
    if len(modResp.ComplianceViolations) > 0 {
        // Handle different types of violations
        violations := modResp.ComplianceViolations
        
        if contains(violations, "drug_use") {
            fmt.Println("Drug-related content detected")
        }
        
        if contains(violations, "gore") || contains(violations, "murder") {
            fmt.Println("Violent content detected")
        }
        
        logViolation(userContent, violations)
        
        return &ModerationResult{
            Allowed:    false,
            Violations: violations,
            Message:    "Content violates our community guidelines",
        }, nil
    }
    
    return &ModerationResult{
        Allowed: true,
        Content: modResp.Choices[0].Message.Content,
    }, nil
}

func contains(slice []string, item string) bool {
    for _, s := range slice {
        if s == item {
            return true
        }
    }
    return false
}

func logViolation(content string, violations []string) {
    fmt.Printf("Violation logged: %s - Content: %s...\n", 
        strings.Join(violations, ", "), 
        content[:min(100, len(content))])
}

func min(a, b int) int {
    if a < b {
        return a
    }
    return b
}

Batch Content Moderation

For applications that need to check multiple pieces of content:

async function batchModerationCheck(contentArray) {
  const results = await Promise.allSettled(
    contentArray.map(content => 
      handleContentModeration(content)
    )
  );
  
  return results.map((result, index) => ({
    index,
    content: contentArray[index],
    result: result.status === 'fulfilled' ? result.value : { error: result.reason }
  }));
}

// Usage
const contentToCheck = [
  "This is normal content",
  "This might contain violations",
  "Another piece of content to check"
];

const batchResults = await batchModerationCheck(contentToCheck);
batchResults.forEach(({ index, content, result }) => {
  if (result.allowed) {
    console.log(`Content ${index}: Approved`);
  } else {
    console.log(`Content ${index}: Rejected - ${result.violations?.join(', ')}`);
  }
});

import asyncio
import aiohttp

async def batch_moderation_check(content_array):
    async with aiohttp.ClientSession() as session:
        tasks = [
            check_content_async(session, content) 
            for content in content_array
        ]
        results = await asyncio.gather(*tasks, return_exceptions=True)
    
    return [
        {
            "index": i,
            "content": content_array[i],
            "result": result if not isinstance(result, Exception) else {"error": str(result)}
        }
        for i, result in enumerate(results)
    ]

async def check_content_async(session, content):
    url = "https://api.animusai.co/v2/chat/completions"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {API_KEY}"
    }
    
    payload = {
        "model": "vivian-llama3.1-70b-1.0-fp8",
        "messages": [{"role": "user", "content": content}],
        "compliance": True
    }
    
    async with session.post(url, headers=headers, json=payload) as response:
        data = await response.json()
        
        if "compliance_violations" in data and data["compliance_violations"]:
            return {
                "allowed": False,
                "violations": data["compliance_violations"]
            }
        
        return {"allowed": True}

# Usage
content_to_check = [
    "This is normal content",
    "This might contain violations", 
    "Another piece of content to check"
]

batch_results = asyncio.run(batch_moderation_check(content_to_check))
for item in batch_results:
    if item["result"].get("allowed"):
        print(f"Content {item['index']}: Approved")
    else:
        violations = item["result"].get("violations", [])
        print(f"Content {item['index']}: Rejected - {', '.join(violations)}")

Best Practices

For effective content moderation in your applications:

Always enable compliance: Set compliance: true for all user-generated content
Implement appropriate responses: Create user-friendly notifications when content is flagged
Pair with frontend filters: Implement basic filtering on the client side to reduce API calls for obvious violations
Handle violations gracefully: Provide constructive feedback to users when their content is flagged
Review edge cases: Periodically review flagged content to understand common violations in your application
Log violations: Keep records for analysis and improvement of your moderation system
Implement appeals process: Allow users to appeal moderation decisions when appropriate

Error Handling

Implement robust error handling for moderation requests:

async function safeContentModeration(content) {
  try {
    const response = await fetch('https://api.animusai.co/v2/chat/completions', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${process.env.ANIMUS_API_KEY}`
      },
      body: JSON.stringify({
        model: "vivian-llama3.1-70b-1.0-fp8",
        messages: [{ role: "user", content: content }],
        compliance: true
      })
    });

    if (!response.ok) {
      throw new Error(`HTTP error! status: ${response.status}`);
    }

    const data = await response.json();
    return {
      success: true,
      violations: data.compliance_violations || [],
      content: data.choices[0].message.content
    };

  } catch (error) {
    console.error('Moderation check failed:', error);
    return {
      success: false,
      error: error.message,
      // Fail safe - assume content needs review
      violations: ['moderation_error']
    };
  }
}

import requests
from requests.exceptions import RequestException

def safe_content_moderation(content):
    try:
        url = "https://api.animusai.co/v2/chat/completions"
        headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {API_KEY}"
        }
        
        payload = {
            "model": "vivian-llama3.1-70b-1.0-fp8",
            "messages": [{"role": "user", "content": content}],
            "compliance": True
        }
        
        response = requests.post(url, headers=headers, json=payload, timeout=30)
        response.raise_for_status()
        
        data = response.json()
        return {
            "success": True,
            "violations": data.get("compliance_violations", []),
            "content": data["choices"][0]["message"]["content"]
        }
        
    except RequestException as e:
        print(f"Moderation check failed: {e}")
        return {
            "success": False,
            "error": str(e),
            # Fail safe - assume content needs review
            "violations": ["moderation_error"]
        }

Integration Examples

Web Application Integration

// Example: Chat application with real-time moderation
class ChatModerator {
  constructor(apiKey) {
    this.apiKey = apiKey;
  }

  async moderateMessage(message) {
    const result = await this.checkCompliance(message);
    
    if (!result.allowed) {
      return {
        blocked: true,
        reason: this.getViolationMessage(result.violations),
        violations: result.violations
      };
    }

    return {
      blocked: false,
      content: result.content
    };
  }

  getViolationMessage(violations) {
    const messages = {
      'drug_use': 'Messages about drug use are not allowed.',
      'gore': 'Graphic violent content is not permitted.',
      'murder': 'Content promoting violence is prohibited.',
      // Add more specific messages
    };

    const specificViolations = violations
      .map(v => messages[v])
      .filter(Boolean);

    return specificViolations.length > 0 
      ? specificViolations.join(' ')
      : 'Your message violates our community guidelines.';
  }
}

// Usage in chat application
const moderator = new ChatModerator(process.env.ANIMUS_API_KEY);

async function handleUserMessage(userMessage) {
  const moderationResult = await moderator.moderateMessage(userMessage);
  
  if (moderationResult.blocked) {
    showUserWarning(moderationResult.reason);
    return;
  }

  // Process the approved message
  displayMessage(moderationResult.content);
}

Next Steps

Text Generation

Learn how to generate text with built-in moderation

Vision

Understand vision capabilities and content analysis

Webhooks

Set up webhooks for automated moderation workflows

API Reference

Complete API documentation and reference

Get Started

SDK Features

Advanced SDK

REST API Integration

Models

Using Content Moderation

Response Format

Content Violation Categories

Advanced Implementation

Handling Different Violation Types

Batch Content Moderation

Best Practices

Error Handling

Integration Examples

Web Application Integration

Next Steps

Text Generation

Vision

Webhooks

API Reference

​Using Content Moderation

​Response Format

​Content Violation Categories

​Advanced Implementation

​Handling Different Violation Types

​Batch Content Moderation

​Best Practices

​Error Handling

​Integration Examples

​Web Application Integration

​Next Steps

Text Generation

Vision

Webhooks

API Reference

Using Content Moderation

Response Format

Content Violation Categories

Advanced Implementation

Handling Different Violation Types

Batch Content Moderation

Best Practices

Error Handling

Integration Examples

Web Application Integration

Next Steps