Thank you, that did the trick!

I modified some of the fields, so here is the unified trigger/test that worked for me:

Here is the trigger:

trigger ContactRollUpOnAccounts on Contact (after delete, after insert, after update) {

    set<ID> AccountIds = new set<ID>();
    set<ID> conIDs = new Set<ID>();
    Set<Id> OldAId = new Set<Id>(); 
   
    if(trigger.isInsert){  
        for(Contact c : trigger.new){
            if(c.AccountId!=null)
            AccountIds.add(c.AccountId);
        }
    }
 
    if(Trigger.isUpdate){    
        for(Contact opp : Trigger.new){         
            if(opp.AccountId != Trigger.oldMap.get(opp.id).AccountId){ 
                AccountIds.add(opp.AccountId); 
                OldAId.add(Trigger.oldMap.get(opp.id).AccountId); 
            }
        }
    }

    if(trigger.isDelete){
        for(Contact c : trigger.old){
            if(c.AccountId!=null)
            AccountIds.add(c.AccountId);
        }
    }
 
    /* for new Account Contacts */
    if(AccountIds!=Null){
    List<AggregateResult> aggs = [select AccountId, count(ID) ContactCount from Contact where AccountId in : AccountIds Group By AccountId];
    Map<Id,Integer> conCountMap = new Map<Id,Integer>();
    for (AggregateResult agg : aggs) {
        conCountMap.put(string.valueOf(agg.get('AccountId')),Integer.valueOf(agg.get('ContactCount')));
    } 
 
    List<Account> AccountsToUpdate = new  List<Account>();
    if(AccountIds != null){
        for(Id acctId : AccountIds ){
            Account newAcct = new Account(id=acctId);
            newAcct.Contact_Count__c = conCountMap.get(acctId);  
            AccountsToUpdate.add(newAcct);
        } 
    }
    update AccountsToUpdate;
 }   

    /* for old Account Contacts */
    if(OldAId!=Null){ 
    List<AggregateResult> aggsOld = [select AccountId, count(ID) ContactCount from Contact where AccountId in : OldAId Group By AccountId]; 
    Map<Id,Integer> conCountMap = new Map<Id,Integer>();
    for (AggregateResult agg : aggsOld) {
        conCountMap.put(string.valueOf(agg.get('AccountId')),Integer.valueOf(agg.get('ContactCount')));
    } 
    List<Account> AccountsToUpdateOld = new  List<Account>();

    if(OldAId!= null){
        for(Id acctId : OldAId){
            Account newAcct = new Account(id=acctId);
            newAcct.Contact_Count__c = conCountMap.get(acctId);  
            AccountsToUpdateOld.add(newAcct);
        } 
    }

    update AccountsToUpdateOld;
        
    }  
}

And the test class:

@isTest

private class ContactRollUpOnAccounts_TestClass{

    public static testMethod void validateContactRollUpOnAccounts() {

        //Create Account
		Account ac = new Account(
			Name='Test Account',
			Initial_Lead_Source__c='Cold Call',
			Initial_Lead_Source_Detail__c='Cold Call',
			Account_Type__c='Prospect',
			Account_Status__c='Not Contacted'
			);
		insert ac;

        //Create Account 2
		Account ac2 = new Account(
			Name='Test Account 2',
			Initial_Lead_Source__c='Cold Call',
			Initial_Lead_Source_Detail__c='Cold Call',
			Account_Type__c='Prospect',
			Account_Status__c='Not Contacted'
			);
		insert ac2;

        //Create Contact 1
        Contact ct1 = new Contact(
                LastName='Test Contact 1',
                Email='testcontact1@email.com',
                AccountID= ac.ID
				);
        insert ct1;
		
		// Verify
        ac = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac.id];
        System.assertEquals(1,ac.Contact_Count__c);

        //Create Contact 2
        Contact ct2 = new Contact(
                LastName='Test Contact 2',
                Email='testcontact2@email.com',
                AccountID= ac.ID
				);
        insert ct2;

		// Verify
        ac = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac.id];
        System.assertEquals(2,ac.Contact_Count__c);
        
        //Delete Contact 1
		delete ct1;

		// Verify
        ac = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac.id];
        System.assertEquals(1,ac.Contact_Count__c);

		//Remove Contact 2
		ct2.AccountID = ac2.ID;
        update ct2;

        // Verify
        ac = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac.id];
        System.assertEquals(null,ac.Contact_Count__c);
        ac2 = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac2.id];
        System.assertEquals(1,ac2.Contact_Count__c);


		//Attach Contact 2
        ct2.AccountID = ac.ID;
        update ct2;

		// Verify
        ac = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac.id];
        System.assertEquals(1,ac.Contact_Count__c);
        ac2 = [SELECT ID, Contact_Count__c FROM Account WHERE ID = :ac2.id];
        System.assertEquals(null,ac2.Contact_Count__c);
    }
}