<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0202</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2004-07-20</periodOfReport>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0001108520</issuerCik>
        <issuerName>WHITTIER ENERGY CORP</issuerName>
        <issuerTradingSymbol>WHIT</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001263541</rptOwnerCik>
            <rptOwnerName>JEFFS JAMES A</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>350 SOUTH WESTGATE AVE.</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>LOS ANGELES</rptOwnerCity>
            <rptOwnerState>CA</rptOwnerState>
            <rptOwnerZipCode>90049</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>1</isOther>
            <officerTitle></officerTitle>
            <otherText>Chairman of the Board</otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <derivativeTable>
        <derivativeTransaction>
            <securityTitle>
                <value>Nonqualified Stock Option (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>1.75</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2004-07-20</value>
                <footnoteId id="F1"/>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>A</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>30000</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>0</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2005-01-08</value>
                <footnoteId id="F2"/>
            </exerciseDate>
            <expirationDate>
                <value>2009-01-08</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>Common Stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>30000</value>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>30000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value></value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
    </derivativeTable>

    <footnotes>
        <footnote id="F1">The stock options were granted on January 8, 2004, subject to the approval by the stockholders of Whittier Energy Corporation of the Whittier Energy Corporation Long Term Incentive Plan, under which the options were granted.  The stockholders of Whittier Energy Corporation approved the Plan on July 20, 2004.</footnote>
        <footnote id="F2">The options vest in three equal installments beginning on January 8, 2005.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ James A. Jeffs</signatureName>
        <signatureDate>2004-07-21</signatureDate>
    </ownerSignature>
</ownershipDocument>

